]> git.lizzy.rs Git - rust.git/blob - src/shims/intrinsics.rs
rustup
[rust.git] / src / shims / intrinsics.rs
1 use std::iter;
2
3 use log::trace;
4
5 use rustc_attr as attr;
6 use rustc_ast::ast::FloatTy;
7 use rustc_middle::{mir, ty};
8 use rustc_middle::ty::layout::IntegerExt;
9 use rustc_apfloat::{Float, Round};
10 use rustc_target::abi::{Align, Integer, LayoutOf};
11 use rustc_span::symbol::sym;
12
13 use crate::*;
14 use helpers::check_arg_count;
15
16 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
17 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
18     fn call_intrinsic(
19         &mut self,
20         instance: ty::Instance<'tcx>,
21         args: &[OpTy<'tcx, Tag>],
22         ret: Option<(PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
23         _unwind: Option<mir::BasicBlock>,
24     ) -> InterpResult<'tcx> {
25         let this = self.eval_context_mut();
26         let intrinsic_name = this.tcx.item_name(instance.def_id());
27         // We want to overwrite some of the intrinsic implementations that CTFE uses.
28         let prefer_miri_intrinsic = match intrinsic_name {
29             sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => true,
30             _ => false,
31         };
32
33         if !prefer_miri_intrinsic && this.emulate_intrinsic(instance, args, ret)? {
34             return Ok(());
35         }
36
37         // All supported intrinsics have a return place.
38         let intrinsic_name = &*intrinsic_name.as_str();
39         let (dest, ret) = match ret {
40             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
41             Some(p) => p,
42         };
43
44         // Then handle terminating intrinsics.
45         match intrinsic_name {
46             // Miri overwriting CTFE intrinsics.
47             "ptr_guaranteed_eq" => {
48                 let &[left, right] = check_arg_count(args)?;
49                 let left = this.read_immediate(left)?;
50                 let right = this.read_immediate(right)?;
51                 this.binop_ignore_overflow(mir::BinOp::Eq, left, right, dest)?;
52             }
53             "ptr_guaranteed_ne" => {
54                 let &[left, right] = check_arg_count(args)?;
55                 let left = this.read_immediate(left)?;
56                 let right = this.read_immediate(right)?;
57                 this.binop_ignore_overflow(mir::BinOp::Ne, left, right, dest)?;
58             }
59
60             // Raw memory accesses
61             #[rustfmt::skip]
62             | "copy"
63             | "copy_nonoverlapping"
64             => {
65                 let &[src, dest, count] = check_arg_count(args)?;
66                 let elem_ty = instance.substs.type_at(0);
67                 let elem_layout = this.layout_of(elem_ty)?;
68                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
69                 let elem_align = elem_layout.align.abi;
70
71                 let size = elem_layout.size.checked_mul(count, this)
72                     .ok_or_else(|| err_ub_format!("overflow computing total size of `{}`", intrinsic_name))?;
73                 let src = this.read_scalar(src)?.check_init()?;
74                 let src = this.memory.check_ptr_access(src, size, elem_align)?;
75                 let dest = this.read_scalar(dest)?.check_init()?;
76                 let dest = this.memory.check_ptr_access(dest, size, elem_align)?;
77
78                 if let (Some(src), Some(dest)) = (src, dest) {
79                     this.memory.copy(
80                         src,
81                         dest,
82                         size,
83                         intrinsic_name.ends_with("_nonoverlapping"),
84                     )?;
85                 }
86             }
87
88             "move_val_init" => {
89                 let &[place, dest] = check_arg_count(args)?;
90                 let place = this.deref_operand(place)?;
91                 this.copy_op(dest, place.into())?;
92             }
93
94             "volatile_load" => {
95                 let &[place] = check_arg_count(args)?;
96                 let place = this.deref_operand(place)?;
97                 this.copy_op(place.into(), dest)?;
98             }
99             "volatile_store" => {
100                 let &[place, dest] = check_arg_count(args)?;
101                 let place = this.deref_operand(place)?;
102                 this.copy_op(dest, place.into())?;
103             }
104
105             "write_bytes" => {
106                 let &[ptr, val_byte, count] = check_arg_count(args)?;
107                 let ty = instance.substs.type_at(0);
108                 let ty_layout = this.layout_of(ty)?;
109                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
110                 let ptr = this.read_scalar(ptr)?.check_init()?;
111                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
112                 let byte_count = ty_layout.size.checked_mul(count, this)
113                     .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?;
114                 this.memory
115                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
116             }
117
118             // Floating-point operations
119             #[rustfmt::skip]
120             | "sinf32"
121             | "fabsf32"
122             | "cosf32"
123             | "sqrtf32"
124             | "expf32"
125             | "exp2f32"
126             | "logf32"
127             | "log10f32"
128             | "log2f32"
129             | "floorf32"
130             | "ceilf32"
131             | "truncf32"
132             | "roundf32"
133             => {
134                 let &[f] = check_arg_count(args)?;
135                 // FIXME: Using host floats.
136                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
137                 let f = match intrinsic_name {
138                     "sinf32" => f.sin(),
139                     "fabsf32" => f.abs(),
140                     "cosf32" => f.cos(),
141                     "sqrtf32" => f.sqrt(),
142                     "expf32" => f.exp(),
143                     "exp2f32" => f.exp2(),
144                     "logf32" => f.ln(),
145                     "log10f32" => f.log10(),
146                     "log2f32" => f.log2(),
147                     "floorf32" => f.floor(),
148                     "ceilf32" => f.ceil(),
149                     "truncf32" => f.trunc(),
150                     "roundf32" => f.round(),
151                     _ => bug!(),
152                 };
153                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
154             }
155
156             #[rustfmt::skip]
157             | "sinf64"
158             | "fabsf64"
159             | "cosf64"
160             | "sqrtf64"
161             | "expf64"
162             | "exp2f64"
163             | "logf64"
164             | "log10f64"
165             | "log2f64"
166             | "floorf64"
167             | "ceilf64"
168             | "truncf64"
169             | "roundf64"
170             => {
171                 let &[f] = check_arg_count(args)?;
172                 // FIXME: Using host floats.
173                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
174                 let f = match intrinsic_name {
175                     "sinf64" => f.sin(),
176                     "fabsf64" => f.abs(),
177                     "cosf64" => f.cos(),
178                     "sqrtf64" => f.sqrt(),
179                     "expf64" => f.exp(),
180                     "exp2f64" => f.exp2(),
181                     "logf64" => f.ln(),
182                     "log10f64" => f.log10(),
183                     "log2f64" => f.log2(),
184                     "floorf64" => f.floor(),
185                     "ceilf64" => f.ceil(),
186                     "truncf64" => f.trunc(),
187                     "roundf64" => f.round(),
188                     _ => bug!(),
189                 };
190                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
191             }
192
193             #[rustfmt::skip]
194             | "fadd_fast"
195             | "fsub_fast"
196             | "fmul_fast"
197             | "fdiv_fast"
198             | "frem_fast"
199             => {
200                 let &[a, b] = check_arg_count(args)?;
201                 let a = this.read_immediate(a)?;
202                 let b = this.read_immediate(b)?;
203                 let op = match intrinsic_name {
204                     "fadd_fast" => mir::BinOp::Add,
205                     "fsub_fast" => mir::BinOp::Sub,
206                     "fmul_fast" => mir::BinOp::Mul,
207                     "fdiv_fast" => mir::BinOp::Div,
208                     "frem_fast" => mir::BinOp::Rem,
209                     _ => bug!(),
210                 };
211                 this.binop_ignore_overflow(op, a, b, dest)?;
212             }
213
214             #[rustfmt::skip]
215             | "minnumf32"
216             | "maxnumf32"
217             | "copysignf32"
218             => {
219                 let &[a, b] = check_arg_count(args)?;
220                 let a = this.read_scalar(a)?.to_f32()?;
221                 let b = this.read_scalar(b)?.to_f32()?;
222                 let res = match intrinsic_name {
223                     "minnumf32" => a.min(b),
224                     "maxnumf32" => a.max(b),
225                     "copysignf32" => a.copy_sign(b),
226                     _ => bug!(),
227                 };
228                 this.write_scalar(Scalar::from_f32(res), dest)?;
229             }
230
231             #[rustfmt::skip]
232             | "minnumf64"
233             | "maxnumf64"
234             | "copysignf64"
235             => {
236                 let &[a, b] = check_arg_count(args)?;
237                 let a = this.read_scalar(a)?.to_f64()?;
238                 let b = this.read_scalar(b)?.to_f64()?;
239                 let res = match intrinsic_name {
240                     "minnumf64" => a.min(b),
241                     "maxnumf64" => a.max(b),
242                     "copysignf64" => a.copy_sign(b),
243                     _ => bug!(),
244                 };
245                 this.write_scalar(Scalar::from_f64(res), dest)?;
246             }
247
248             "powf32" => {
249                 let &[f, f2] = check_arg_count(args)?;
250                 // FIXME: Using host floats.
251                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
252                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
253                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
254             }
255
256             "powf64" => {
257                 let &[f, f2] = check_arg_count(args)?;
258                 // FIXME: Using host floats.
259                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
260                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
261                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
262             }
263
264             "fmaf32" => {
265                 let &[a, b, c] = check_arg_count(args)?;
266                 let a = this.read_scalar(a)?.to_f32()?;
267                 let b = this.read_scalar(b)?.to_f32()?;
268                 let c = this.read_scalar(c)?.to_f32()?;
269                 let res = a.mul_add(b, c).value;
270                 this.write_scalar(Scalar::from_f32(res), dest)?;
271             }
272
273             "fmaf64" => {
274                 let &[a, b, c] = check_arg_count(args)?;
275                 let a = this.read_scalar(a)?.to_f64()?;
276                 let b = this.read_scalar(b)?.to_f64()?;
277                 let c = this.read_scalar(c)?.to_f64()?;
278                 let res = a.mul_add(b, c).value;
279                 this.write_scalar(Scalar::from_f64(res), dest)?;
280             }
281
282             "powif32" => {
283                 let &[f, i] = check_arg_count(args)?;
284                 // FIXME: Using host floats.
285                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
286                 let i = this.read_scalar(i)?.to_i32()?;
287                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
288             }
289
290             "powif64" => {
291                 let &[f, i] = check_arg_count(args)?;
292                 // FIXME: Using host floats.
293                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
294                 let i = this.read_scalar(i)?.to_i32()?;
295                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
296             }
297
298             "float_to_int_unchecked" => {
299                 let &[val] = check_arg_count(args)?;
300                 let val = this.read_immediate(val)?;
301
302                 let res = match val.layout.ty.kind {
303                     ty::Float(FloatTy::F32) => {
304                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?
305                     }
306                     ty::Float(FloatTy::F64) => {
307                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?
308                     }
309                     _ => bug!("`float_to_int_unchecked` called with non-float input type {:?}", val.layout.ty),
310                 };
311
312                 this.write_scalar(res, dest)?;
313             }
314
315             // Atomic operations
316             #[rustfmt::skip]
317             | "atomic_load"
318             | "atomic_load_relaxed"
319             | "atomic_load_acq"
320             => {
321                 let &[place] = check_arg_count(args)?;
322                 let place = this.deref_operand(place)?;
323                 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
324
325                 // Check alignment requirements. Atomics must always be aligned to their size,
326                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
327                 // be 8-aligned).
328                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
329                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
330
331                 this.write_scalar(val, dest)?;
332             }
333
334             #[rustfmt::skip]
335             | "atomic_store"
336             | "atomic_store_relaxed"
337             | "atomic_store_rel"
338             => {
339                 let &[place, val] = check_arg_count(args)?;
340                 let place = this.deref_operand(place)?;
341                 let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
342
343                 // Check alignment requirements. Atomics must always be aligned to their size,
344                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
345                 // be 8-aligned).
346                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
347                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
348
349                 this.write_scalar(val, place.into())?;
350             }
351
352             #[rustfmt::skip]
353             | "atomic_fence_acq"
354             | "atomic_fence_rel"
355             | "atomic_fence_acqrel"
356             | "atomic_fence"
357             | "atomic_singlethreadfence_acq"
358             | "atomic_singlethreadfence_rel"
359             | "atomic_singlethreadfence_acqrel"
360             | "atomic_singlethreadfence"
361             => {
362                 let &[] = check_arg_count(args)?;
363                 // FIXME: this will become relevant once we try to detect data races.
364             }
365
366             _ if intrinsic_name.starts_with("atomic_xchg") => {
367                 let &[place, new] = check_arg_count(args)?;
368                 let place = this.deref_operand(place)?;
369                 let new = this.read_scalar(new)?;
370                 let old = this.read_scalar(place.into())?;
371
372                 // Check alignment requirements. Atomics must always be aligned to their size,
373                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
374                 // be 8-aligned).
375                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
376                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
377
378                 this.write_scalar(old, dest)?; // old value is returned
379                 this.write_scalar(new, place.into())?;
380             }
381
382             _ if intrinsic_name.starts_with("atomic_cxchg") => {
383                 let &[place, expect_old, new] = check_arg_count(args)?;
384                 let place = this.deref_operand(place)?;
385                 let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
386                 let new = this.read_scalar(new)?;
387                 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
388
389                 // Check alignment requirements. Atomics must always be aligned to their size,
390                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
391                 // be 8-aligned).
392                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
393                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
394
395                 // `binary_op` will bail if either of them is not a scalar.
396                 let eq = this.overflowing_binary_op(mir::BinOp::Eq, old, expect_old)?.0;
397                 let res = Immediate::ScalarPair(old.to_scalar_or_uninit(), eq.into());
398                 // Return old value.
399                 this.write_immediate(res, dest)?;
400                 // Update ptr depending on comparison.
401                 if eq.to_bool()? {
402                     this.write_scalar(new, place.into())?;
403                 }
404             }
405
406             #[rustfmt::skip]
407             | "atomic_or"
408             | "atomic_or_acq"
409             | "atomic_or_rel"
410             | "atomic_or_acqrel"
411             | "atomic_or_relaxed"
412             | "atomic_xor"
413             | "atomic_xor_acq"
414             | "atomic_xor_rel"
415             | "atomic_xor_acqrel"
416             | "atomic_xor_relaxed"
417             | "atomic_and"
418             | "atomic_and_acq"
419             | "atomic_and_rel"
420             | "atomic_and_acqrel"
421             | "atomic_and_relaxed"
422             | "atomic_nand"
423             | "atomic_nand_acq"
424             | "atomic_nand_rel"
425             | "atomic_nand_acqrel"
426             | "atomic_nand_relaxed"
427             | "atomic_xadd"
428             | "atomic_xadd_acq"
429             | "atomic_xadd_rel"
430             | "atomic_xadd_acqrel"
431             | "atomic_xadd_relaxed"
432             | "atomic_xsub"
433             | "atomic_xsub_acq"
434             | "atomic_xsub_rel"
435             | "atomic_xsub_acqrel"
436             | "atomic_xsub_relaxed"
437             => {
438                 let &[place, rhs] = check_arg_count(args)?;
439                 let place = this.deref_operand(place)?;
440                 if !place.layout.ty.is_integral() {
441                     bug!("Atomic arithmetic operations only work on integer types");
442                 }
443                 let rhs = this.read_immediate(rhs)?;
444                 let old = this.read_immediate(place.into())?;
445
446                 // Check alignment requirements. Atomics must always be aligned to their size,
447                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
448                 // be 8-aligned).
449                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
450                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
451
452                 this.write_immediate(*old, dest)?; // old value is returned
453                 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
454                     "or" => (mir::BinOp::BitOr, false),
455                     "xor" => (mir::BinOp::BitXor, false),
456                     "and" => (mir::BinOp::BitAnd, false),
457                     "xadd" => (mir::BinOp::Add, false),
458                     "xsub" => (mir::BinOp::Sub, false),
459                     "nand" => (mir::BinOp::BitAnd, true),
460                     _ => bug!(),
461                 };
462                 // Atomics wrap around on overflow.
463                 let val = this.binary_op(op, old, rhs)?;
464                 let val = if neg { this.unary_op(mir::UnOp::Not, val)? } else { val };
465                 this.write_immediate(*val, place.into())?;
466             }
467
468             // Query type information
469             "assert_inhabited" |
470             "assert_zero_valid" |
471             "assert_uninit_valid" => {
472                 let &[] = check_arg_count(args)?;
473                 let ty = instance.substs.type_at(0);
474                 let layout = this.layout_of(ty)?;
475                 // Abort here because the caller might not be panic safe.
476                 if layout.abi.is_uninhabited() {
477                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to instantiate uninhabited type `{}`", ty))))
478                 }
479                 if intrinsic_name == "assert_zero_valid" && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap() {
480                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to zero-initialize type `{}`, which is invalid", ty))))
481                 }
482                 if intrinsic_name == "assert_uninit_valid" && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap() {
483                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to leave type `{}` uninitialized, which is invalid", ty))))
484                 }
485             }
486
487             "min_align_of_val" => {
488                 let &[mplace] = check_arg_count(args)?;
489                 let mplace = this.deref_operand(mplace)?;
490                 let (_, align) = this
491                     .size_and_align_of_mplace(mplace)?
492                     .expect("size_of_val called on extern type");
493                 this.write_scalar(Scalar::from_machine_usize(align.bytes(), this), dest)?;
494             }
495
496             "size_of_val" => {
497                 let &[mplace] = check_arg_count(args)?;
498                 let mplace = this.deref_operand(mplace)?;
499                 let (size, _) = this
500                     .size_and_align_of_mplace(mplace)?
501                     .expect("size_of_val called on extern type");
502                 this.write_scalar(Scalar::from_machine_usize(size.bytes(), this), dest)?;
503             }
504
505             // Other
506             "assume" => {
507                 let &[cond] = check_arg_count(args)?;
508                 let cond = this.read_scalar(cond)?.check_init()?.to_bool()?;
509                 if !cond {
510                     throw_ub_format!("`assume` intrinsic called with `false`");
511                 }
512             }
513
514             "exact_div" => {
515                 let &[num, denom] = check_arg_count(args)?;
516                 this.exact_div(this.read_immediate(num)?, this.read_immediate(denom)?, dest)?;
517             }
518
519             "forget" => {
520                 // We get an argument... and forget about it.
521                 let &[_] = check_arg_count(args)?;
522             }
523
524             "try" => return this.handle_try(args, dest, ret),
525
526             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
527         }
528
529         trace!("{:?}", this.dump_place(*dest));
530         this.go_to_block(ret);
531         Ok(())
532     }
533
534     fn float_to_int_unchecked<F>(
535         &self,
536         f: F,
537         dest_ty: ty::Ty<'tcx>,
538     ) -> InterpResult<'tcx, Scalar<Tag>>
539     where
540         F: Float + Into<Scalar<Tag>>
541     {
542         let this = self.eval_context_ref();
543
544         // Step 1: cut off the fractional part of `f`. The result of this is
545         // guaranteed to be precisely representable in IEEE floats.
546         let f = f.round_to_integral(Round::TowardZero).value;
547
548         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
549         Ok(match dest_ty.kind {
550             // Unsigned
551             ty::Uint(t) => {
552                 let size = Integer::from_attr(this, attr::IntType::UnsignedInt(t)).size();
553                 let res = f.to_u128(size.bits_usize());
554                 if res.status.is_empty() {
555                     // No status flags means there was no further rounding or other loss of precision.
556                     Scalar::from_uint(res.value, size)
557                 } else {
558                     // `f` was not representable in this integer type.
559                     throw_ub_format!(
560                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
561                         f, dest_ty,
562                     );
563                 }
564             }
565             // Signed
566             ty::Int(t) => {
567                 let size = Integer::from_attr(this, attr::IntType::SignedInt(t)).size();
568                 let res = f.to_i128(size.bits_usize());
569                 if res.status.is_empty() {
570                     // No status flags means there was no further rounding or other loss of precision.
571                     Scalar::from_int(res.value, size)
572                 } else {
573                     // `f` was not representable in this integer type.
574                     throw_ub_format!(
575                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
576                         f, dest_ty,
577                     );
578                 }
579             }
580             // Nothing else
581             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
582         })
583     }
584 }