]> git.lizzy.rs Git - rust.git/blob - src/shims/intrinsics.rs
fix diagnostics printing when triggered during TLS dtor scheduling
[rust.git] / src / shims / intrinsics.rs
1 use std::iter;
2
3 use rustc_attr as attr;
4 use rustc_ast::ast::FloatTy;
5 use rustc_middle::{mir, ty};
6 use rustc_middle::ty::layout::IntegerExt;
7 use rustc_apfloat::{Float, Round};
8 use rustc_target::abi::{Align, Integer, LayoutOf};
9 use rustc_span::symbol::sym;
10
11 use crate::*;
12 use helpers::check_arg_count;
13
14 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
15 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
16     fn call_intrinsic(
17         &mut self,
18         instance: ty::Instance<'tcx>,
19         args: &[OpTy<'tcx, Tag>],
20         ret: Option<(PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
21         _unwind: Option<mir::BasicBlock>,
22     ) -> InterpResult<'tcx> {
23         let this = self.eval_context_mut();
24         let intrinsic_name = this.tcx.item_name(instance.def_id());
25         // We want to overwrite some of the intrinsic implementations that CTFE uses.
26         let prefer_miri_intrinsic = match intrinsic_name {
27             sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => true,
28             _ => false,
29         };
30
31         if !prefer_miri_intrinsic && this.emulate_intrinsic(instance, args, ret)? {
32             return Ok(());
33         }
34
35         // All supported intrinsics have a return place.
36         let intrinsic_name = &*intrinsic_name.as_str();
37         let (dest, ret) = match ret {
38             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
39             Some(p) => p,
40         };
41
42         // Then handle terminating intrinsics.
43         match intrinsic_name {
44             // Miri overwriting CTFE intrinsics.
45             "ptr_guaranteed_eq" => {
46                 let &[left, right] = check_arg_count(args)?;
47                 let left = this.read_immediate(left)?;
48                 let right = this.read_immediate(right)?;
49                 this.binop_ignore_overflow(mir::BinOp::Eq, left, right, dest)?;
50             }
51             "ptr_guaranteed_ne" => {
52                 let &[left, right] = check_arg_count(args)?;
53                 let left = this.read_immediate(left)?;
54                 let right = this.read_immediate(right)?;
55                 this.binop_ignore_overflow(mir::BinOp::Ne, left, right, dest)?;
56             }
57
58             // Raw memory accesses
59             #[rustfmt::skip]
60             | "copy"
61             | "copy_nonoverlapping"
62             => {
63                 let &[src, dest, count] = check_arg_count(args)?;
64                 let elem_ty = instance.substs.type_at(0);
65                 let elem_layout = this.layout_of(elem_ty)?;
66                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
67                 let elem_align = elem_layout.align.abi;
68
69                 let size = elem_layout.size.checked_mul(count, this)
70                     .ok_or_else(|| err_ub_format!("overflow computing total size of `{}`", intrinsic_name))?;
71                 let src = this.read_scalar(src)?.check_init()?;
72                 let src = this.memory.check_ptr_access(src, size, elem_align)?;
73                 let dest = this.read_scalar(dest)?.check_init()?;
74                 let dest = this.memory.check_ptr_access(dest, size, elem_align)?;
75
76                 if let (Some(src), Some(dest)) = (src, dest) {
77                     this.memory.copy(
78                         src,
79                         dest,
80                         size,
81                         intrinsic_name.ends_with("_nonoverlapping"),
82                     )?;
83                 }
84             }
85
86             "move_val_init" => {
87                 let &[place, dest] = check_arg_count(args)?;
88                 let place = this.deref_operand(place)?;
89                 this.copy_op(dest, place.into())?;
90             }
91
92             "volatile_load" => {
93                 let &[place] = check_arg_count(args)?;
94                 let place = this.deref_operand(place)?;
95                 this.copy_op(place.into(), dest)?;
96             }
97             "volatile_store" => {
98                 let &[place, dest] = check_arg_count(args)?;
99                 let place = this.deref_operand(place)?;
100                 this.copy_op(dest, place.into())?;
101             }
102
103             "write_bytes" => {
104                 let &[ptr, val_byte, count] = check_arg_count(args)?;
105                 let ty = instance.substs.type_at(0);
106                 let ty_layout = this.layout_of(ty)?;
107                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
108                 let ptr = this.read_scalar(ptr)?.check_init()?;
109                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
110                 let byte_count = ty_layout.size.checked_mul(count, this)
111                     .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?;
112                 this.memory
113                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
114             }
115
116             // Floating-point operations
117             #[rustfmt::skip]
118             | "sinf32"
119             | "fabsf32"
120             | "cosf32"
121             | "sqrtf32"
122             | "expf32"
123             | "exp2f32"
124             | "logf32"
125             | "log10f32"
126             | "log2f32"
127             | "floorf32"
128             | "ceilf32"
129             | "truncf32"
130             | "roundf32"
131             => {
132                 let &[f] = check_arg_count(args)?;
133                 // FIXME: Using host floats.
134                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
135                 let f = match intrinsic_name {
136                     "sinf32" => f.sin(),
137                     "fabsf32" => f.abs(),
138                     "cosf32" => f.cos(),
139                     "sqrtf32" => f.sqrt(),
140                     "expf32" => f.exp(),
141                     "exp2f32" => f.exp2(),
142                     "logf32" => f.ln(),
143                     "log10f32" => f.log10(),
144                     "log2f32" => f.log2(),
145                     "floorf32" => f.floor(),
146                     "ceilf32" => f.ceil(),
147                     "truncf32" => f.trunc(),
148                     "roundf32" => f.round(),
149                     _ => bug!(),
150                 };
151                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
152             }
153
154             #[rustfmt::skip]
155             | "sinf64"
156             | "fabsf64"
157             | "cosf64"
158             | "sqrtf64"
159             | "expf64"
160             | "exp2f64"
161             | "logf64"
162             | "log10f64"
163             | "log2f64"
164             | "floorf64"
165             | "ceilf64"
166             | "truncf64"
167             | "roundf64"
168             => {
169                 let &[f] = check_arg_count(args)?;
170                 // FIXME: Using host floats.
171                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
172                 let f = match intrinsic_name {
173                     "sinf64" => f.sin(),
174                     "fabsf64" => f.abs(),
175                     "cosf64" => f.cos(),
176                     "sqrtf64" => f.sqrt(),
177                     "expf64" => f.exp(),
178                     "exp2f64" => f.exp2(),
179                     "logf64" => f.ln(),
180                     "log10f64" => f.log10(),
181                     "log2f64" => f.log2(),
182                     "floorf64" => f.floor(),
183                     "ceilf64" => f.ceil(),
184                     "truncf64" => f.trunc(),
185                     "roundf64" => f.round(),
186                     _ => bug!(),
187                 };
188                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
189             }
190
191             #[rustfmt::skip]
192             | "fadd_fast"
193             | "fsub_fast"
194             | "fmul_fast"
195             | "fdiv_fast"
196             | "frem_fast"
197             => {
198                 let &[a, b] = check_arg_count(args)?;
199                 let a = this.read_immediate(a)?;
200                 let b = this.read_immediate(b)?;
201                 let op = match intrinsic_name {
202                     "fadd_fast" => mir::BinOp::Add,
203                     "fsub_fast" => mir::BinOp::Sub,
204                     "fmul_fast" => mir::BinOp::Mul,
205                     "fdiv_fast" => mir::BinOp::Div,
206                     "frem_fast" => mir::BinOp::Rem,
207                     _ => bug!(),
208                 };
209                 this.binop_ignore_overflow(op, a, b, dest)?;
210             }
211
212             #[rustfmt::skip]
213             | "minnumf32"
214             | "maxnumf32"
215             | "copysignf32"
216             => {
217                 let &[a, b] = check_arg_count(args)?;
218                 let a = this.read_scalar(a)?.to_f32()?;
219                 let b = this.read_scalar(b)?.to_f32()?;
220                 let res = match intrinsic_name {
221                     "minnumf32" => a.min(b),
222                     "maxnumf32" => a.max(b),
223                     "copysignf32" => a.copy_sign(b),
224                     _ => bug!(),
225                 };
226                 this.write_scalar(Scalar::from_f32(res), dest)?;
227             }
228
229             #[rustfmt::skip]
230             | "minnumf64"
231             | "maxnumf64"
232             | "copysignf64"
233             => {
234                 let &[a, b] = check_arg_count(args)?;
235                 let a = this.read_scalar(a)?.to_f64()?;
236                 let b = this.read_scalar(b)?.to_f64()?;
237                 let res = match intrinsic_name {
238                     "minnumf64" => a.min(b),
239                     "maxnumf64" => a.max(b),
240                     "copysignf64" => a.copy_sign(b),
241                     _ => bug!(),
242                 };
243                 this.write_scalar(Scalar::from_f64(res), dest)?;
244             }
245
246             "powf32" => {
247                 let &[f, f2] = check_arg_count(args)?;
248                 // FIXME: Using host floats.
249                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
250                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
251                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
252             }
253
254             "powf64" => {
255                 let &[f, f2] = check_arg_count(args)?;
256                 // FIXME: Using host floats.
257                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
258                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
259                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
260             }
261
262             "fmaf32" => {
263                 let &[a, b, c] = check_arg_count(args)?;
264                 let a = this.read_scalar(a)?.to_f32()?;
265                 let b = this.read_scalar(b)?.to_f32()?;
266                 let c = this.read_scalar(c)?.to_f32()?;
267                 let res = a.mul_add(b, c).value;
268                 this.write_scalar(Scalar::from_f32(res), dest)?;
269             }
270
271             "fmaf64" => {
272                 let &[a, b, c] = check_arg_count(args)?;
273                 let a = this.read_scalar(a)?.to_f64()?;
274                 let b = this.read_scalar(b)?.to_f64()?;
275                 let c = this.read_scalar(c)?.to_f64()?;
276                 let res = a.mul_add(b, c).value;
277                 this.write_scalar(Scalar::from_f64(res), dest)?;
278             }
279
280             "powif32" => {
281                 let &[f, i] = check_arg_count(args)?;
282                 // FIXME: Using host floats.
283                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
284                 let i = this.read_scalar(i)?.to_i32()?;
285                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
286             }
287
288             "powif64" => {
289                 let &[f, i] = check_arg_count(args)?;
290                 // FIXME: Using host floats.
291                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
292                 let i = this.read_scalar(i)?.to_i32()?;
293                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
294             }
295
296             "float_to_int_unchecked" => {
297                 let &[val] = check_arg_count(args)?;
298                 let val = this.read_immediate(val)?;
299
300                 let res = match val.layout.ty.kind {
301                     ty::Float(FloatTy::F32) => {
302                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?
303                     }
304                     ty::Float(FloatTy::F64) => {
305                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?
306                     }
307                     _ => bug!("`float_to_int_unchecked` called with non-float input type {:?}", val.layout.ty),
308                 };
309
310                 this.write_scalar(res, dest)?;
311             }
312
313             // Atomic operations
314             #[rustfmt::skip]
315             | "atomic_load"
316             | "atomic_load_relaxed"
317             | "atomic_load_acq"
318             => {
319                 let &[place] = check_arg_count(args)?;
320                 let place = this.deref_operand(place)?;
321                 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
322
323                 // Check alignment requirements. Atomics must always be aligned to their size,
324                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
325                 // be 8-aligned).
326                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
327                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
328
329                 this.write_scalar(val, dest)?;
330             }
331
332             #[rustfmt::skip]
333             | "atomic_store"
334             | "atomic_store_relaxed"
335             | "atomic_store_rel"
336             => {
337                 let &[place, val] = check_arg_count(args)?;
338                 let place = this.deref_operand(place)?;
339                 let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
340
341                 // Check alignment requirements. Atomics must always be aligned to their size,
342                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
343                 // be 8-aligned).
344                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
345                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
346
347                 this.write_scalar(val, place.into())?;
348             }
349
350             #[rustfmt::skip]
351             | "atomic_fence_acq"
352             | "atomic_fence_rel"
353             | "atomic_fence_acqrel"
354             | "atomic_fence"
355             | "atomic_singlethreadfence_acq"
356             | "atomic_singlethreadfence_rel"
357             | "atomic_singlethreadfence_acqrel"
358             | "atomic_singlethreadfence"
359             => {
360                 let &[] = check_arg_count(args)?;
361                 // FIXME: this will become relevant once we try to detect data races.
362             }
363
364             _ if intrinsic_name.starts_with("atomic_xchg") => {
365                 let &[place, new] = check_arg_count(args)?;
366                 let place = this.deref_operand(place)?;
367                 let new = this.read_scalar(new)?;
368                 let old = this.read_scalar(place.into())?;
369
370                 // Check alignment requirements. Atomics must always be aligned to their size,
371                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
372                 // be 8-aligned).
373                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
374                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
375
376                 this.write_scalar(old, dest)?; // old value is returned
377                 this.write_scalar(new, place.into())?;
378             }
379
380             _ if intrinsic_name.starts_with("atomic_cxchg") => {
381                 let &[place, expect_old, new] = check_arg_count(args)?;
382                 let place = this.deref_operand(place)?;
383                 let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
384                 let new = this.read_scalar(new)?;
385                 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
386
387                 // Check alignment requirements. Atomics must always be aligned to their size,
388                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
389                 // be 8-aligned).
390                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
391                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
392
393                 // `binary_op` will bail if either of them is not a scalar.
394                 let eq = this.overflowing_binary_op(mir::BinOp::Eq, old, expect_old)?.0;
395                 let res = Immediate::ScalarPair(old.to_scalar_or_uninit(), eq.into());
396                 // Return old value.
397                 this.write_immediate(res, dest)?;
398                 // Update ptr depending on comparison.
399                 if eq.to_bool()? {
400                     this.write_scalar(new, place.into())?;
401                 }
402             }
403
404             #[rustfmt::skip]
405             | "atomic_or"
406             | "atomic_or_acq"
407             | "atomic_or_rel"
408             | "atomic_or_acqrel"
409             | "atomic_or_relaxed"
410             | "atomic_xor"
411             | "atomic_xor_acq"
412             | "atomic_xor_rel"
413             | "atomic_xor_acqrel"
414             | "atomic_xor_relaxed"
415             | "atomic_and"
416             | "atomic_and_acq"
417             | "atomic_and_rel"
418             | "atomic_and_acqrel"
419             | "atomic_and_relaxed"
420             | "atomic_nand"
421             | "atomic_nand_acq"
422             | "atomic_nand_rel"
423             | "atomic_nand_acqrel"
424             | "atomic_nand_relaxed"
425             | "atomic_xadd"
426             | "atomic_xadd_acq"
427             | "atomic_xadd_rel"
428             | "atomic_xadd_acqrel"
429             | "atomic_xadd_relaxed"
430             | "atomic_xsub"
431             | "atomic_xsub_acq"
432             | "atomic_xsub_rel"
433             | "atomic_xsub_acqrel"
434             | "atomic_xsub_relaxed"
435             => {
436                 let &[place, rhs] = check_arg_count(args)?;
437                 let place = this.deref_operand(place)?;
438                 if !place.layout.ty.is_integral() {
439                     bug!("Atomic arithmetic operations only work on integer types");
440                 }
441                 let rhs = this.read_immediate(rhs)?;
442                 let old = this.read_immediate(place.into())?;
443
444                 // Check alignment requirements. Atomics must always be aligned to their size,
445                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
446                 // be 8-aligned).
447                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
448                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
449
450                 this.write_immediate(*old, dest)?; // old value is returned
451                 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
452                     "or" => (mir::BinOp::BitOr, false),
453                     "xor" => (mir::BinOp::BitXor, false),
454                     "and" => (mir::BinOp::BitAnd, false),
455                     "xadd" => (mir::BinOp::Add, false),
456                     "xsub" => (mir::BinOp::Sub, false),
457                     "nand" => (mir::BinOp::BitAnd, true),
458                     _ => bug!(),
459                 };
460                 // Atomics wrap around on overflow.
461                 let val = this.binary_op(op, old, rhs)?;
462                 let val = if neg { this.unary_op(mir::UnOp::Not, val)? } else { val };
463                 this.write_immediate(*val, place.into())?;
464             }
465
466             // Query type information
467             "assert_inhabited" |
468             "assert_zero_valid" |
469             "assert_uninit_valid" => {
470                 let &[] = check_arg_count(args)?;
471                 let ty = instance.substs.type_at(0);
472                 let layout = this.layout_of(ty)?;
473                 // Abort here because the caller might not be panic safe.
474                 if layout.abi.is_uninhabited() {
475                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to instantiate uninhabited type `{}`", ty))))
476                 }
477                 if intrinsic_name == "assert_zero_valid" && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap() {
478                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to zero-initialize type `{}`, which is invalid", ty))))
479                 }
480                 if intrinsic_name == "assert_uninit_valid" && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap() {
481                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to leave type `{}` uninitialized, which is invalid", ty))))
482                 }
483             }
484
485             "min_align_of_val" => {
486                 let &[mplace] = check_arg_count(args)?;
487                 let mplace = this.deref_operand(mplace)?;
488                 let (_, align) = this
489                     .size_and_align_of_mplace(mplace)?
490                     .expect("size_of_val called on extern type");
491                 this.write_scalar(Scalar::from_machine_usize(align.bytes(), this), dest)?;
492             }
493
494             "size_of_val" => {
495                 let &[mplace] = check_arg_count(args)?;
496                 let mplace = this.deref_operand(mplace)?;
497                 let (size, _) = this
498                     .size_and_align_of_mplace(mplace)?
499                     .expect("size_of_val called on extern type");
500                 this.write_scalar(Scalar::from_machine_usize(size.bytes(), this), dest)?;
501             }
502
503             // Other
504             "assume" => {
505                 let &[cond] = check_arg_count(args)?;
506                 let cond = this.read_scalar(cond)?.check_init()?.to_bool()?;
507                 if !cond {
508                     throw_ub_format!("`assume` intrinsic called with `false`");
509                 }
510             }
511
512             "exact_div" => {
513                 let &[num, denom] = check_arg_count(args)?;
514                 this.exact_div(this.read_immediate(num)?, this.read_immediate(denom)?, dest)?;
515             }
516
517             "forget" => {
518                 // We get an argument... and forget about it.
519                 let &[_] = check_arg_count(args)?;
520             }
521
522             "try" => return this.handle_try(args, dest, ret),
523
524             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
525         }
526
527         this.dump_place(*dest);
528         this.go_to_block(ret);
529         Ok(())
530     }
531
532     fn float_to_int_unchecked<F>(
533         &self,
534         f: F,
535         dest_ty: ty::Ty<'tcx>,
536     ) -> InterpResult<'tcx, Scalar<Tag>>
537     where
538         F: Float + Into<Scalar<Tag>>
539     {
540         let this = self.eval_context_ref();
541
542         // Step 1: cut off the fractional part of `f`. The result of this is
543         // guaranteed to be precisely representable in IEEE floats.
544         let f = f.round_to_integral(Round::TowardZero).value;
545
546         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
547         Ok(match dest_ty.kind {
548             // Unsigned
549             ty::Uint(t) => {
550                 let size = Integer::from_attr(this, attr::IntType::UnsignedInt(t)).size();
551                 let res = f.to_u128(size.bits_usize());
552                 if res.status.is_empty() {
553                     // No status flags means there was no further rounding or other loss of precision.
554                     Scalar::from_uint(res.value, size)
555                 } else {
556                     // `f` was not representable in this integer type.
557                     throw_ub_format!(
558                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
559                         f, dest_ty,
560                     );
561                 }
562             }
563             // Signed
564             ty::Int(t) => {
565                 let size = Integer::from_attr(this, attr::IntType::SignedInt(t)).size();
566                 let res = f.to_i128(size.bits_usize());
567                 if res.status.is_empty() {
568                     // No status flags means there was no further rounding or other loss of precision.
569                     Scalar::from_int(res.value, size)
570                 } else {
571                     // `f` was not representable in this integer type.
572                     throw_ub_format!(
573                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
574                         f, dest_ty,
575                     );
576                 }
577             }
578             // Nothing else
579             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
580         })
581     }
582 }