]> git.lizzy.rs Git - rust.git/blob - src/shims/intrinsics.rs
Support `gettimeofday` on more than macos
[rust.git] / src / shims / intrinsics.rs
1 use std::iter;
2
3 use log::trace;
4
5 use rustc_apfloat::{Float, Round};
6 use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
8 use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size};
9
10 use crate::*;
11 use helpers::check_arg_count;
12
13 pub enum AtomicOp {
14     MirOp(mir::BinOp, bool),
15     Max,
16     Min,
17 }
18
19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
21     fn call_intrinsic(
22         &mut self,
23         instance: ty::Instance<'tcx>,
24         args: &[OpTy<'tcx, Tag>],
25         dest: &PlaceTy<'tcx, Tag>,
26         ret: Option<mir::BasicBlock>,
27         _unwind: StackPopUnwind,
28     ) -> InterpResult<'tcx> {
29         let this = self.eval_context_mut();
30
31         if this.emulate_intrinsic(instance, args, dest, ret)? {
32             return Ok(());
33         }
34
35         // All supported intrinsics have a return place.
36         let intrinsic_name = this.tcx.item_name(instance.def_id());
37         let intrinsic_name = intrinsic_name.as_str();
38         let ret = match ret {
39             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
40             Some(p) => p,
41         };
42
43         // Then handle terminating intrinsics.
44         match intrinsic_name {
45             // Miri overwriting CTFE intrinsics.
46             "ptr_guaranteed_eq" => {
47                 let [left, right] = check_arg_count(args)?;
48                 let left = this.read_immediate(left)?;
49                 let right = this.read_immediate(right)?;
50                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
51             }
52             "ptr_guaranteed_ne" => {
53                 let [left, right] = check_arg_count(args)?;
54                 let left = this.read_immediate(left)?;
55                 let right = this.read_immediate(right)?;
56                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
57             }
58             "const_allocate" => {
59                 // For now, for compatibility with the run-time implementation of this, we just return null.
60                 // See <https://github.com/rust-lang/rust/issues/93935>.
61                 this.write_null(dest)?;
62             }
63             "const_deallocate" => {
64                 // complete NOP
65             }
66
67             // Raw memory accesses
68             "volatile_load" => {
69                 let [place] = check_arg_count(args)?;
70                 let place = this.deref_operand(place)?;
71                 this.copy_op(&place.into(), dest)?;
72             }
73             "volatile_store" => {
74                 let [place, dest] = check_arg_count(args)?;
75                 let place = this.deref_operand(place)?;
76                 this.copy_op(dest, &place.into())?;
77             }
78
79             "write_bytes" | "volatile_set_memory" => {
80                 let [ptr, val_byte, count] = check_arg_count(args)?;
81                 let ty = instance.substs.type_at(0);
82                 let ty_layout = this.layout_of(ty)?;
83                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
84                 let ptr = this.read_pointer(ptr)?;
85                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
86                 // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
87                 // but no actual allocation can be big enough for the difference to be noticeable.
88                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
89                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
90                 })?;
91                 this.write_bytes_ptr(
92                     ptr,
93                     iter::repeat(val_byte).take(byte_count.bytes() as usize),
94                 )?;
95             }
96
97             // Floating-point operations
98             "fabsf32" => {
99                 let [f] = check_arg_count(args)?;
100                 let f = this.read_scalar(f)?.to_f32()?;
101                 // Can be implemented in soft-floats.
102                 this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
103             }
104             "fabsf64" => {
105                 let [f] = check_arg_count(args)?;
106                 let f = this.read_scalar(f)?.to_f64()?;
107                 // Can be implemented in soft-floats.
108                 this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
109             }
110             #[rustfmt::skip]
111             | "sinf32"
112             | "cosf32"
113             | "sqrtf32"
114             | "expf32"
115             | "exp2f32"
116             | "logf32"
117             | "log10f32"
118             | "log2f32"
119             | "floorf32"
120             | "ceilf32"
121             | "truncf32"
122             | "roundf32"
123             => {
124                 let [f] = check_arg_count(args)?;
125                 // FIXME: Using host floats.
126                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
127                 let f = match intrinsic_name {
128                     "sinf32" => f.sin(),
129                     "cosf32" => f.cos(),
130                     "sqrtf32" => f.sqrt(),
131                     "expf32" => f.exp(),
132                     "exp2f32" => f.exp2(),
133                     "logf32" => f.ln(),
134                     "log10f32" => f.log10(),
135                     "log2f32" => f.log2(),
136                     "floorf32" => f.floor(),
137                     "ceilf32" => f.ceil(),
138                     "truncf32" => f.trunc(),
139                     "roundf32" => f.round(),
140                     _ => bug!(),
141                 };
142                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
143             }
144
145             #[rustfmt::skip]
146             | "sinf64"
147             | "cosf64"
148             | "sqrtf64"
149             | "expf64"
150             | "exp2f64"
151             | "logf64"
152             | "log10f64"
153             | "log2f64"
154             | "floorf64"
155             | "ceilf64"
156             | "truncf64"
157             | "roundf64"
158             => {
159                 let [f] = check_arg_count(args)?;
160                 // FIXME: Using host floats.
161                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
162                 let f = match intrinsic_name {
163                     "sinf64" => f.sin(),
164                     "cosf64" => f.cos(),
165                     "sqrtf64" => f.sqrt(),
166                     "expf64" => f.exp(),
167                     "exp2f64" => f.exp2(),
168                     "logf64" => f.ln(),
169                     "log10f64" => f.log10(),
170                     "log2f64" => f.log2(),
171                     "floorf64" => f.floor(),
172                     "ceilf64" => f.ceil(),
173                     "truncf64" => f.trunc(),
174                     "roundf64" => f.round(),
175                     _ => bug!(),
176                 };
177                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
178             }
179
180             #[rustfmt::skip]
181             | "fadd_fast"
182             | "fsub_fast"
183             | "fmul_fast"
184             | "fdiv_fast"
185             | "frem_fast"
186             => {
187                 let [a, b] = check_arg_count(args)?;
188                 let a = this.read_immediate(a)?;
189                 let b = this.read_immediate(b)?;
190                 let op = match intrinsic_name {
191                     "fadd_fast" => mir::BinOp::Add,
192                     "fsub_fast" => mir::BinOp::Sub,
193                     "fmul_fast" => mir::BinOp::Mul,
194                     "fdiv_fast" => mir::BinOp::Div,
195                     "frem_fast" => mir::BinOp::Rem,
196                     _ => bug!(),
197                 };
198                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
199                     Ok(match x.layout.ty.kind() {
200                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
201                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
202                         _ => bug!(
203                             "`{}` called with non-float input type {:?}",
204                             intrinsic_name,
205                             x.layout.ty
206                         ),
207                     })
208                 };
209                 match (float_finite(a)?, float_finite(b)?) {
210                     (false, false) => throw_ub_format!(
211                         "`{}` intrinsic called with non-finite value as both parameters",
212                         intrinsic_name,
213                     ),
214                     (false, _) => throw_ub_format!(
215                         "`{}` intrinsic called with non-finite value as first parameter",
216                         intrinsic_name,
217                     ),
218                     (_, false) => throw_ub_format!(
219                         "`{}` intrinsic called with non-finite value as second parameter",
220                         intrinsic_name,
221                     ),
222                     _ => {}
223                 }
224                 this.binop_ignore_overflow(op, &a, &b, dest)?;
225             }
226
227             #[rustfmt::skip]
228             | "minnumf32"
229             | "maxnumf32"
230             | "copysignf32"
231             => {
232                 let [a, b] = check_arg_count(args)?;
233                 let a = this.read_scalar(a)?.to_f32()?;
234                 let b = this.read_scalar(b)?.to_f32()?;
235                 let res = match intrinsic_name {
236                     "minnumf32" => a.min(b),
237                     "maxnumf32" => a.max(b),
238                     "copysignf32" => a.copy_sign(b),
239                     _ => bug!(),
240                 };
241                 this.write_scalar(Scalar::from_f32(res), dest)?;
242             }
243
244             #[rustfmt::skip]
245             | "minnumf64"
246             | "maxnumf64"
247             | "copysignf64"
248             => {
249                 let [a, b] = check_arg_count(args)?;
250                 let a = this.read_scalar(a)?.to_f64()?;
251                 let b = this.read_scalar(b)?.to_f64()?;
252                 let res = match intrinsic_name {
253                     "minnumf64" => a.min(b),
254                     "maxnumf64" => a.max(b),
255                     "copysignf64" => a.copy_sign(b),
256                     _ => bug!(),
257                 };
258                 this.write_scalar(Scalar::from_f64(res), dest)?;
259             }
260
261             "powf32" => {
262                 let [f, f2] = check_arg_count(args)?;
263                 // FIXME: Using host floats.
264                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
265                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
266                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
267             }
268
269             "powf64" => {
270                 let [f, f2] = check_arg_count(args)?;
271                 // FIXME: Using host floats.
272                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
273                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
274                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
275             }
276
277             "fmaf32" => {
278                 let [a, b, c] = check_arg_count(args)?;
279                 let a = this.read_scalar(a)?.to_f32()?;
280                 let b = this.read_scalar(b)?.to_f32()?;
281                 let c = this.read_scalar(c)?.to_f32()?;
282                 let res = a.mul_add(b, c).value;
283                 this.write_scalar(Scalar::from_f32(res), dest)?;
284             }
285
286             "fmaf64" => {
287                 let [a, b, c] = check_arg_count(args)?;
288                 let a = this.read_scalar(a)?.to_f64()?;
289                 let b = this.read_scalar(b)?.to_f64()?;
290                 let c = this.read_scalar(c)?.to_f64()?;
291                 let res = a.mul_add(b, c).value;
292                 this.write_scalar(Scalar::from_f64(res), dest)?;
293             }
294
295             "powif32" => {
296                 let [f, i] = check_arg_count(args)?;
297                 // FIXME: Using host floats.
298                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
299                 let i = this.read_scalar(i)?.to_i32()?;
300                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
301             }
302
303             "powif64" => {
304                 let [f, i] = check_arg_count(args)?;
305                 // FIXME: Using host floats.
306                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
307                 let i = this.read_scalar(i)?.to_i32()?;
308                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
309             }
310
311             "float_to_int_unchecked" => {
312                 let [val] = check_arg_count(args)?;
313                 let val = this.read_immediate(val)?;
314
315                 let res = match val.layout.ty.kind() {
316                     ty::Float(FloatTy::F32) =>
317                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
318                     ty::Float(FloatTy::F64) =>
319                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
320                     _ =>
321                         bug!(
322                             "`float_to_int_unchecked` called with non-float input type {:?}",
323                             val.layout.ty
324                         ),
325                 };
326
327                 this.write_scalar(res, dest)?;
328             }
329
330             // SIMD operations
331             #[rustfmt::skip]
332             | "simd_neg"
333             | "simd_fabs"
334             | "simd_ceil"
335             | "simd_floor"
336             | "simd_round"
337             | "simd_trunc"
338             | "simd_fsqrt" => {
339                 let [op] = check_arg_count(args)?;
340                 let (op, op_len) = this.operand_to_simd(op)?;
341                 let (dest, dest_len) = this.place_to_simd(dest)?;
342
343                 assert_eq!(dest_len, op_len);
344
345                 #[derive(Copy, Clone)]
346                 enum HostFloatOp {
347                     Ceil,
348                     Floor,
349                     Round,
350                     Trunc,
351                     Sqrt,
352                 }
353                 #[derive(Copy, Clone)]
354                 enum Op {
355                     MirOp(mir::UnOp),
356                     Abs,
357                     HostOp(HostFloatOp),
358                 }
359                 let which = match intrinsic_name {
360                     "simd_neg" => Op::MirOp(mir::UnOp::Neg),
361                     "simd_fabs" => Op::Abs,
362                     "simd_ceil" => Op::HostOp(HostFloatOp::Ceil),
363                     "simd_floor" => Op::HostOp(HostFloatOp::Floor),
364                     "simd_round" => Op::HostOp(HostFloatOp::Round),
365                     "simd_trunc" => Op::HostOp(HostFloatOp::Trunc),
366                     "simd_fsqrt" => Op::HostOp(HostFloatOp::Sqrt),
367                     _ => unreachable!(),
368                 };
369
370                 for i in 0..dest_len {
371                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
372                     let dest = this.mplace_index(&dest, i)?;
373                     let val = match which {
374                         Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
375                         Op::Abs => {
376                             // Works for f32 and f64.
377                             let ty::Float(float_ty) = op.layout.ty.kind() else {
378                                 bug!("{} operand is not a float", intrinsic_name)
379                             };
380                             let op = op.to_scalar()?;
381                             match float_ty {
382                                 FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
383                                 FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
384                             }
385                         }
386                         Op::HostOp(host_op) => {
387                             let ty::Float(float_ty) = op.layout.ty.kind() else {
388                                 bug!("{} operand is not a float", intrinsic_name)
389                             };
390                             // FIXME using host floats
391                             match float_ty {
392                                 FloatTy::F32 => {
393                                     let f = f32::from_bits(op.to_scalar()?.to_u32()?);
394                                     let res = match host_op {
395                                         HostFloatOp::Ceil => f.ceil(),
396                                         HostFloatOp::Floor => f.floor(),
397                                         HostFloatOp::Round => f.round(),
398                                         HostFloatOp::Trunc => f.trunc(),
399                                         HostFloatOp::Sqrt => f.sqrt(),
400                                     };
401                                     Scalar::from_u32(res.to_bits())
402                                 }
403                                 FloatTy::F64 => {
404                                     let f = f64::from_bits(op.to_scalar()?.to_u64()?);
405                                     let res = match host_op {
406                                         HostFloatOp::Ceil => f.ceil(),
407                                         HostFloatOp::Floor => f.floor(),
408                                         HostFloatOp::Round => f.round(),
409                                         HostFloatOp::Trunc => f.trunc(),
410                                         HostFloatOp::Sqrt => f.sqrt(),
411                                     };
412                                     Scalar::from_u64(res.to_bits())
413                                 }
414                             }
415
416                         }
417                     };
418                     this.write_scalar(val, &dest.into())?;
419                 }
420             }
421             #[rustfmt::skip]
422             | "simd_add"
423             | "simd_sub"
424             | "simd_mul"
425             | "simd_div"
426             | "simd_rem"
427             | "simd_shl"
428             | "simd_shr"
429             | "simd_and"
430             | "simd_or"
431             | "simd_xor"
432             | "simd_eq"
433             | "simd_ne"
434             | "simd_lt"
435             | "simd_le"
436             | "simd_gt"
437             | "simd_ge"
438             | "simd_fmax"
439             | "simd_fmin"
440             | "simd_saturating_add"
441             | "simd_saturating_sub"
442             | "simd_arith_offset" => {
443                 use mir::BinOp;
444
445                 let [left, right] = check_arg_count(args)?;
446                 let (left, left_len) = this.operand_to_simd(left)?;
447                 let (right, right_len) = this.operand_to_simd(right)?;
448                 let (dest, dest_len) = this.place_to_simd(dest)?;
449
450                 assert_eq!(dest_len, left_len);
451                 assert_eq!(dest_len, right_len);
452
453                 enum Op {
454                     MirOp(BinOp),
455                     SaturatingOp(BinOp),
456                     FMax,
457                     FMin,
458                     WrappingOffset,
459                 }
460                 let which = match intrinsic_name {
461                     "simd_add" => Op::MirOp(BinOp::Add),
462                     "simd_sub" => Op::MirOp(BinOp::Sub),
463                     "simd_mul" => Op::MirOp(BinOp::Mul),
464                     "simd_div" => Op::MirOp(BinOp::Div),
465                     "simd_rem" => Op::MirOp(BinOp::Rem),
466                     "simd_shl" => Op::MirOp(BinOp::Shl),
467                     "simd_shr" => Op::MirOp(BinOp::Shr),
468                     "simd_and" => Op::MirOp(BinOp::BitAnd),
469                     "simd_or" => Op::MirOp(BinOp::BitOr),
470                     "simd_xor" => Op::MirOp(BinOp::BitXor),
471                     "simd_eq" => Op::MirOp(BinOp::Eq),
472                     "simd_ne" => Op::MirOp(BinOp::Ne),
473                     "simd_lt" => Op::MirOp(BinOp::Lt),
474                     "simd_le" => Op::MirOp(BinOp::Le),
475                     "simd_gt" => Op::MirOp(BinOp::Gt),
476                     "simd_ge" => Op::MirOp(BinOp::Ge),
477                     "simd_fmax" => Op::FMax,
478                     "simd_fmin" => Op::FMin,
479                     "simd_saturating_add" => Op::SaturatingOp(BinOp::Add),
480                     "simd_saturating_sub" => Op::SaturatingOp(BinOp::Sub),
481                     "simd_arith_offset" => Op::WrappingOffset,
482                     _ => unreachable!(),
483                 };
484
485                 for i in 0..dest_len {
486                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
487                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
488                     let dest = this.mplace_index(&dest, i)?;
489                     let val = match which {
490                         Op::MirOp(mir_op) => {
491                             let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
492                             if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
493                                 // Shifts have extra UB as SIMD operations that the MIR binop does not have.
494                                 // See <https://github.com/rust-lang/rust/issues/91237>.
495                                 if overflowed {
496                                     let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
497                                     throw_ub_format!("overflowing shift by {} in `{}` in SIMD lane {}", r_val, intrinsic_name, i);
498                                 }
499                             }
500                             if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
501                                 // Special handling for boolean-returning operations
502                                 assert_eq!(ty, this.tcx.types.bool);
503                                 let val = val.to_bool().unwrap();
504                                 bool_to_simd_element(val, dest.layout.size)
505                             } else {
506                                 assert_ne!(ty, this.tcx.types.bool);
507                                 assert_eq!(ty, dest.layout.ty);
508                                 val
509                             }
510                         }
511                         Op::SaturatingOp(mir_op) => {
512                             this.saturating_arith(mir_op, &left, &right)?
513                         }
514                         Op::WrappingOffset => {
515                             let ptr = this.scalar_to_ptr(left.to_scalar()?)?;
516                             let offset_count = right.to_scalar()?.to_machine_isize(this)?;
517                             let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty;
518
519                             let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap();
520                             let offset_bytes = offset_count.wrapping_mul(pointee_size);
521                             let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this);
522                             Scalar::from_maybe_pointer(offset_ptr, this)
523                         }
524                         Op::FMax => {
525                             fmax_op(&left, &right)?
526                         }
527                         Op::FMin => {
528                             fmin_op(&left, &right)?
529                         }
530                     };
531                     this.write_scalar(val, &dest.into())?;
532                 }
533             }
534             "simd_fma" => {
535                 let [a, b, c] = check_arg_count(args)?;
536                 let (a, a_len) = this.operand_to_simd(a)?;
537                 let (b, b_len) = this.operand_to_simd(b)?;
538                 let (c, c_len) = this.operand_to_simd(c)?;
539                 let (dest, dest_len) = this.place_to_simd(dest)?;
540
541                 assert_eq!(dest_len, a_len);
542                 assert_eq!(dest_len, b_len);
543                 assert_eq!(dest_len, c_len);
544
545                 for i in 0..dest_len {
546                     let a = this.read_immediate(&this.mplace_index(&a, i)?.into())?.to_scalar()?;
547                     let b = this.read_immediate(&this.mplace_index(&b, i)?.into())?.to_scalar()?;
548                     let c = this.read_immediate(&this.mplace_index(&c, i)?.into())?.to_scalar()?;
549                     let dest = this.mplace_index(&dest, i)?;
550
551                     // Works for f32 and f64.
552                     let ty::Float(float_ty) = dest.layout.ty.kind() else {
553                         bug!("{} operand is not a float", intrinsic_name)
554                     };
555                     let val = match float_ty {
556                         FloatTy::F32 =>
557                             Scalar::from_f32(a.to_f32()?.mul_add(b.to_f32()?, c.to_f32()?).value),
558                         FloatTy::F64 =>
559                             Scalar::from_f64(a.to_f64()?.mul_add(b.to_f64()?, c.to_f64()?).value),
560                     };
561                     this.write_scalar(val, &dest.into())?;
562                 }
563             }
564             #[rustfmt::skip]
565             | "simd_reduce_and"
566             | "simd_reduce_or"
567             | "simd_reduce_xor"
568             | "simd_reduce_any"
569             | "simd_reduce_all"
570             | "simd_reduce_max"
571             | "simd_reduce_min" => {
572                 use mir::BinOp;
573
574                 let [op] = check_arg_count(args)?;
575                 let (op, op_len) = this.operand_to_simd(op)?;
576
577                 let imm_from_bool =
578                     |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
579
580                 enum Op {
581                     MirOp(BinOp),
582                     MirOpBool(BinOp),
583                     Max,
584                     Min,
585                 }
586                 let which = match intrinsic_name {
587                     "simd_reduce_and" => Op::MirOp(BinOp::BitAnd),
588                     "simd_reduce_or" => Op::MirOp(BinOp::BitOr),
589                     "simd_reduce_xor" => Op::MirOp(BinOp::BitXor),
590                     "simd_reduce_any" => Op::MirOpBool(BinOp::BitOr),
591                     "simd_reduce_all" => Op::MirOpBool(BinOp::BitAnd),
592                     "simd_reduce_max" => Op::Max,
593                     "simd_reduce_min" => Op::Min,
594                     _ => unreachable!(),
595                 };
596
597                 // Initialize with first lane, then proceed with the rest.
598                 let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
599                 if matches!(which, Op::MirOpBool(_)) {
600                     // Convert to `bool` scalar.
601                     res = imm_from_bool(simd_element_to_bool(res)?);
602                 }
603                 for i in 1..op_len {
604                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
605                     res = match which {
606                         Op::MirOp(mir_op) => {
607                             this.binary_op(mir_op, &res, &op)?
608                         }
609                         Op::MirOpBool(mir_op) => {
610                             let op = imm_from_bool(simd_element_to_bool(op)?);
611                             this.binary_op(mir_op, &res, &op)?
612                         }
613                         Op::Max => {
614                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
615                                 ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
616                             } else {
617                                 // Just boring integers, so NaNs to worry about
618                                 if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
619                                     res
620                                 } else {
621                                     op
622                                 }
623                             }
624                         }
625                         Op::Min => {
626                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
627                                 ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
628                             } else {
629                                 // Just boring integers, so NaNs to worry about
630                                 if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
631                                     res
632                                 } else {
633                                     op
634                                 }
635                             }
636                         }
637                     };
638                 }
639                 this.write_immediate(*res, dest)?;
640             }
641             #[rustfmt::skip]
642             | "simd_reduce_add_ordered"
643             | "simd_reduce_mul_ordered" => {
644                 use mir::BinOp;
645
646                 let [op, init] = check_arg_count(args)?;
647                 let (op, op_len) = this.operand_to_simd(op)?;
648                 let init = this.read_immediate(init)?;
649
650                 let mir_op = match intrinsic_name {
651                     "simd_reduce_add_ordered" => BinOp::Add,
652                     "simd_reduce_mul_ordered" => BinOp::Mul,
653                     _ => unreachable!(),
654                 };
655
656                 let mut res = init;
657                 for i in 0..op_len {
658                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
659                     res = this.binary_op(mir_op, &res, &op)?;
660                 }
661                 this.write_immediate(*res, dest)?;
662             }
663             "simd_select" => {
664                 let [mask, yes, no] = check_arg_count(args)?;
665                 let (mask, mask_len) = this.operand_to_simd(mask)?;
666                 let (yes, yes_len) = this.operand_to_simd(yes)?;
667                 let (no, no_len) = this.operand_to_simd(no)?;
668                 let (dest, dest_len) = this.place_to_simd(dest)?;
669
670                 assert_eq!(dest_len, mask_len);
671                 assert_eq!(dest_len, yes_len);
672                 assert_eq!(dest_len, no_len);
673
674                 for i in 0..dest_len {
675                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
676                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
677                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
678                     let dest = this.mplace_index(&dest, i)?;
679
680                     let val = if simd_element_to_bool(mask)? { yes } else { no };
681                     this.write_immediate(*val, &dest.into())?;
682                 }
683             }
684             "simd_select_bitmask" => {
685                 let [mask, yes, no] = check_arg_count(args)?;
686                 let (yes, yes_len) = this.operand_to_simd(yes)?;
687                 let (no, no_len) = this.operand_to_simd(no)?;
688                 let (dest, dest_len) = this.place_to_simd(dest)?;
689                 let bitmask_len = dest_len.max(8);
690
691                 assert!(mask.layout.ty.is_integral());
692                 assert!(bitmask_len <= 64);
693                 assert_eq!(bitmask_len, mask.layout.size.bits());
694                 assert_eq!(dest_len, yes_len);
695                 assert_eq!(dest_len, no_len);
696
697                 let mask: u64 = this
698                     .read_scalar(mask)?
699                     .check_init()?
700                     .to_bits(mask.layout.size)?
701                     .try_into()
702                     .unwrap();
703                 for i in 0..dest_len {
704                     let mask =
705                         mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
706                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
707                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
708                     let dest = this.mplace_index(&dest, i)?;
709
710                     let val = if mask != 0 { yes } else { no };
711                     this.write_immediate(*val, &dest.into())?;
712                 }
713                 for i in dest_len..bitmask_len {
714                     // If the mask is "padded", ensure that padding is all-zero.
715                     let mask = mask & (1 << i);
716                     if mask != 0 {
717                         throw_ub_format!(
718                             "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits"
719                         );
720                     }
721                 }
722             }
723             #[rustfmt::skip]
724             "simd_cast" | "simd_as" => {
725                 let [op] = check_arg_count(args)?;
726                 let (op, op_len) = this.operand_to_simd(op)?;
727                 let (dest, dest_len) = this.place_to_simd(dest)?;
728
729                 assert_eq!(dest_len, op_len);
730
731                 let safe_cast = intrinsic_name == "simd_as";
732
733                 for i in 0..dest_len {
734                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
735                     let dest = this.mplace_index(&dest, i)?;
736
737                     let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
738                         // Int-to-(int|float): always safe
739                         (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
740                             this.misc_cast(&op, dest.layout.ty)?,
741                         // Float-to-float: always safe
742                         (ty::Float(_), ty::Float(_)) =>
743                             this.misc_cast(&op, dest.layout.ty)?,
744                         // Float-to-int in safe mode
745                         (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
746                             this.misc_cast(&op, dest.layout.ty)?,
747                         // Float-to-int in unchecked mode
748                         (ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
749                             this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
750                         (ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
751                             this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
752                         _ =>
753                             throw_unsup_format!(
754                                 "Unsupported SIMD cast from element type {} to {}",
755                                 op.layout.ty,
756                                 dest.layout.ty
757                             ),
758                     };
759                     this.write_immediate(val, &dest.into())?;
760                 }
761             }
762             "simd_shuffle" => {
763                 let [left, right, index] = check_arg_count(args)?;
764                 let (left, left_len) = this.operand_to_simd(left)?;
765                 let (right, right_len) = this.operand_to_simd(right)?;
766                 let (dest, dest_len) = this.place_to_simd(dest)?;
767
768                 // `index` is an array, not a SIMD type
769                 let ty::Array(_, index_len) = index.layout.ty.kind() else {
770                     bug!("simd_shuffle index argument has non-array type {}", index.layout.ty)
771                 };
772                 let index_len = index_len.eval_usize(*this.tcx, this.param_env());
773
774                 assert_eq!(left_len, right_len);
775                 assert_eq!(index_len, dest_len);
776
777                 for i in 0..dest_len {
778                     let src_index: u64 = this
779                         .read_immediate(&this.operand_index(index, i)?)?
780                         .to_scalar()?
781                         .to_u32()?
782                         .into();
783                     let dest = this.mplace_index(&dest, i)?;
784
785                     let val = if src_index < left_len {
786                         this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
787                     } else if src_index < left_len.checked_add(right_len).unwrap() {
788                         this.read_immediate(
789                             &this.mplace_index(&right, src_index - left_len)?.into(),
790                         )?
791                     } else {
792                         bug!(
793                             "simd_shuffle index {} is out of bounds for 2 vectors of size {}",
794                             src_index,
795                             left_len
796                         );
797                     };
798                     this.write_immediate(*val, &dest.into())?;
799                 }
800             }
801             "simd_gather" => {
802                 let [passthru, ptrs, mask] = check_arg_count(args)?;
803                 let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
804                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
805                 let (mask, mask_len) = this.operand_to_simd(mask)?;
806                 let (dest, dest_len) = this.place_to_simd(dest)?;
807
808                 assert_eq!(dest_len, passthru_len);
809                 assert_eq!(dest_len, ptrs_len);
810                 assert_eq!(dest_len, mask_len);
811
812                 for i in 0..dest_len {
813                     let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
814                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
815                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
816                     let dest = this.mplace_index(&dest, i)?;
817
818                     let val = if simd_element_to_bool(mask)? {
819                         let place = this.deref_operand(&ptr.into())?;
820                         this.read_immediate(&place.into())?
821                     } else {
822                         passthru
823                     };
824                     this.write_immediate(*val, &dest.into())?;
825                 }
826             }
827             "simd_scatter" => {
828                 let [value, ptrs, mask] = check_arg_count(args)?;
829                 let (value, value_len) = this.operand_to_simd(value)?;
830                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
831                 let (mask, mask_len) = this.operand_to_simd(mask)?;
832
833                 assert_eq!(ptrs_len, value_len);
834                 assert_eq!(ptrs_len, mask_len);
835
836                 for i in 0..ptrs_len {
837                     let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
838                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
839                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
840
841                     if simd_element_to_bool(mask)? {
842                         let place = this.deref_operand(&ptr.into())?;
843                         this.write_immediate(*value, &place.into())?;
844                     }
845                 }
846             }
847             "simd_bitmask" => {
848                 let [op] = check_arg_count(args)?;
849                 let (op, op_len) = this.operand_to_simd(op)?;
850                 let bitmask_len = op_len.max(8);
851
852                 assert!(dest.layout.ty.is_integral());
853                 assert!(bitmask_len <= 64);
854                 assert_eq!(bitmask_len, dest.layout.size.bits());
855
856                 let mut res = 0u64;
857                 for i in 0..op_len {
858                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
859                     if simd_element_to_bool(op)? {
860                         res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
861                     }
862                 }
863                 this.write_int(res, dest)?;
864             }
865
866             // Atomic operations
867             "atomic_load_seqcst" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
868             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
869             "atomic_load_acquire" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
870
871             "atomic_store_seqcst" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
872             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
873             "atomic_store_release" => this.atomic_store(args, AtomicWriteOp::Release)?,
874
875             "atomic_fence_acquire" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
876             "atomic_fence_release" => this.atomic_fence(args, AtomicFenceOp::Release)?,
877             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
878             "atomic_fence_seqcst" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
879
880             "atomic_singlethreadfence_acquire" =>
881                 this.compiler_fence(args, AtomicFenceOp::Acquire)?,
882             "atomic_singlethreadfence_release" =>
883                 this.compiler_fence(args, AtomicFenceOp::Release)?,
884             "atomic_singlethreadfence_acqrel" =>
885                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
886             "atomic_singlethreadfence_seqcst" =>
887                 this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
888
889             "atomic_xchg_seqcst" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
890             "atomic_xchg_acquire" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
891             "atomic_xchg_release" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
892             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
893             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
894
895             #[rustfmt::skip]
896             "atomic_cxchg_seqcst_seqcst" =>
897                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
898             #[rustfmt::skip]
899             "atomic_cxchg_acquire_acquire" =>
900                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
901             #[rustfmt::skip]
902             "atomic_cxchg_release_relaxed" =>
903                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
904             #[rustfmt::skip]
905             "atomic_cxchg_acqrel_acquire" =>
906                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
907             #[rustfmt::skip]
908             "atomic_cxchg_relaxed_relaxed" =>
909                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
910             #[rustfmt::skip]
911             "atomic_cxchg_acquire_relaxed" =>
912                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
913             #[rustfmt::skip]
914             "atomic_cxchg_acqrel_relaxed" =>
915                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
916             #[rustfmt::skip]
917             "atomic_cxchg_seqcst_relaxed" =>
918                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
919             #[rustfmt::skip]
920             "atomic_cxchg_seqcst_acquire" =>
921                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
922
923             #[rustfmt::skip]
924             "atomic_cxchgweak_seqcst_seqcst" =>
925                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
926             #[rustfmt::skip]
927             "atomic_cxchgweak_acquire_acquire" =>
928                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
929             #[rustfmt::skip]
930             "atomic_cxchgweak_release_relaxed" =>
931                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
932             #[rustfmt::skip]
933             "atomic_cxchgweak_acqrel_acquire" =>
934                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
935             #[rustfmt::skip]
936             "atomic_cxchgweak_relaxed_relaxed" =>
937                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
938             #[rustfmt::skip]
939             "atomic_cxchgweak_acquire_relaxed" =>
940                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
941             #[rustfmt::skip]
942             "atomic_cxchgweak_acqrel_relaxed" =>
943                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
944             #[rustfmt::skip]
945             "atomic_cxchgweak_seqcst_relaxed" =>
946                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
947             #[rustfmt::skip]
948             "atomic_cxchgweak_seqcst_acquire" =>
949                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
950
951             #[rustfmt::skip]
952             "atomic_or_seqcst" =>
953                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
954             #[rustfmt::skip]
955             "atomic_or_acquire" =>
956                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
957             #[rustfmt::skip]
958             "atomic_or_release" =>
959                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
960             #[rustfmt::skip]
961             "atomic_or_acqrel" =>
962                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
963             #[rustfmt::skip]
964             "atomic_or_relaxed" =>
965                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
966             #[rustfmt::skip]
967             "atomic_xor_seqcst" =>
968                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
969             #[rustfmt::skip]
970             "atomic_xor_acquire" =>
971                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
972             #[rustfmt::skip]
973             "atomic_xor_release" =>
974                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
975             #[rustfmt::skip]
976             "atomic_xor_acqrel" =>
977                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
978             #[rustfmt::skip]
979             "atomic_xor_relaxed" =>
980                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
981             #[rustfmt::skip]
982             "atomic_and_seqcst" =>
983                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
984             #[rustfmt::skip]
985             "atomic_and_acquire" =>
986                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
987             #[rustfmt::skip]
988             "atomic_and_release" =>
989                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
990             #[rustfmt::skip]
991             "atomic_and_acqrel" =>
992                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
993             #[rustfmt::skip]
994             "atomic_and_relaxed" =>
995                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
996             #[rustfmt::skip]
997             "atomic_nand_seqcst" =>
998                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
999             #[rustfmt::skip]
1000             "atomic_nand_acquire" =>
1001                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
1002             #[rustfmt::skip]
1003             "atomic_nand_release" =>
1004                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
1005             #[rustfmt::skip]
1006             "atomic_nand_acqrel" =>
1007                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
1008             #[rustfmt::skip]
1009             "atomic_nand_relaxed" =>
1010                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
1011             #[rustfmt::skip]
1012             "atomic_xadd_seqcst" =>
1013                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
1014             #[rustfmt::skip]
1015             "atomic_xadd_acquire" =>
1016                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
1017             #[rustfmt::skip]
1018             "atomic_xadd_release" =>
1019                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
1020             #[rustfmt::skip]
1021             "atomic_xadd_acqrel" =>
1022                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
1023             #[rustfmt::skip]
1024             "atomic_xadd_relaxed" =>
1025                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
1026             #[rustfmt::skip]
1027             "atomic_xsub_seqcst" =>
1028                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
1029             #[rustfmt::skip]
1030             "atomic_xsub_acquire" =>
1031                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
1032             #[rustfmt::skip]
1033             "atomic_xsub_release" =>
1034                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
1035             #[rustfmt::skip]
1036             "atomic_xsub_acqrel" =>
1037                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
1038             #[rustfmt::skip]
1039             "atomic_xsub_relaxed" =>
1040                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
1041             "atomic_min_seqcst" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1042             "atomic_min_acquire" =>
1043                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1044             "atomic_min_release" =>
1045                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1046             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1047             "atomic_min_relaxed" =>
1048                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1049             "atomic_max_seqcst" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1050             "atomic_max_acquire" =>
1051                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1052             "atomic_max_release" =>
1053                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1054             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1055             "atomic_max_relaxed" =>
1056                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1057             "atomic_umin_seqcst" =>
1058                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1059             "atomic_umin_acquire" =>
1060                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1061             "atomic_umin_release" =>
1062                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1063             "atomic_umin_acqrel" =>
1064                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1065             "atomic_umin_relaxed" =>
1066                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1067             "atomic_umax_seqcst" =>
1068                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1069             "atomic_umax_acquire" =>
1070                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1071             "atomic_umax_release" =>
1072                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1073             "atomic_umax_acqrel" =>
1074                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1075             "atomic_umax_relaxed" =>
1076                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1077
1078             // Other
1079             "exact_div" => {
1080                 let [num, denom] = check_arg_count(args)?;
1081                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
1082             }
1083
1084             "try" => return this.handle_try(args, dest, ret),
1085
1086             "breakpoint" => {
1087                 let [] = check_arg_count(args)?;
1088                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
1089                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
1090             }
1091
1092             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
1093         }
1094
1095         trace!("{:?}", this.dump_place(**dest));
1096         this.go_to_block(ret);
1097         Ok(())
1098     }
1099
1100     fn atomic_load(
1101         &mut self,
1102         args: &[OpTy<'tcx, Tag>],
1103         dest: &PlaceTy<'tcx, Tag>,
1104         atomic: AtomicReadOp,
1105     ) -> InterpResult<'tcx> {
1106         let this = self.eval_context_mut();
1107
1108         let [place] = check_arg_count(args)?;
1109         let place = this.deref_operand(place)?;
1110
1111         // make sure it fits into a scalar; otherwise it cannot be atomic
1112         let val = this.read_scalar_atomic(&place, atomic)?;
1113
1114         // Check alignment requirements. Atomics must always be aligned to their size,
1115         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1116         // be 8-aligned).
1117         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1118         this.check_ptr_access_align(
1119             place.ptr,
1120             place.layout.size,
1121             align,
1122             CheckInAllocMsg::MemoryAccessTest,
1123         )?;
1124         // Perform regular access.
1125         this.write_scalar(val, dest)?;
1126         Ok(())
1127     }
1128
1129     fn atomic_store(
1130         &mut self,
1131         args: &[OpTy<'tcx, Tag>],
1132         atomic: AtomicWriteOp,
1133     ) -> InterpResult<'tcx> {
1134         let this = self.eval_context_mut();
1135
1136         let [place, val] = check_arg_count(args)?;
1137         let place = this.deref_operand(place)?;
1138         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
1139
1140         // Check alignment requirements. Atomics must always be aligned to their size,
1141         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1142         // be 8-aligned).
1143         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1144         this.check_ptr_access_align(
1145             place.ptr,
1146             place.layout.size,
1147             align,
1148             CheckInAllocMsg::MemoryAccessTest,
1149         )?;
1150
1151         // Perform atomic store
1152         this.write_scalar_atomic(val, &place, atomic)?;
1153         Ok(())
1154     }
1155
1156     fn compiler_fence(
1157         &mut self,
1158         args: &[OpTy<'tcx, Tag>],
1159         atomic: AtomicFenceOp,
1160     ) -> InterpResult<'tcx> {
1161         let [] = check_arg_count(args)?;
1162         let _ = atomic;
1163         //FIXME: compiler fences are currently ignored
1164         Ok(())
1165     }
1166
1167     fn atomic_fence(
1168         &mut self,
1169         args: &[OpTy<'tcx, Tag>],
1170         atomic: AtomicFenceOp,
1171     ) -> InterpResult<'tcx> {
1172         let this = self.eval_context_mut();
1173         let [] = check_arg_count(args)?;
1174         this.validate_atomic_fence(atomic)?;
1175         Ok(())
1176     }
1177
1178     fn atomic_op(
1179         &mut self,
1180         args: &[OpTy<'tcx, Tag>],
1181         dest: &PlaceTy<'tcx, Tag>,
1182         atomic_op: AtomicOp,
1183         atomic: AtomicRwOp,
1184     ) -> InterpResult<'tcx> {
1185         let this = self.eval_context_mut();
1186
1187         let [place, rhs] = check_arg_count(args)?;
1188         let place = this.deref_operand(place)?;
1189
1190         if !place.layout.ty.is_integral() {
1191             bug!("Atomic arithmetic operations only work on integer types");
1192         }
1193         let rhs = this.read_immediate(rhs)?;
1194
1195         // Check alignment requirements. Atomics must always be aligned to their size,
1196         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1197         // be 8-aligned).
1198         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1199         this.check_ptr_access_align(
1200             place.ptr,
1201             place.layout.size,
1202             align,
1203             CheckInAllocMsg::MemoryAccessTest,
1204         )?;
1205
1206         match atomic_op {
1207             AtomicOp::Min => {
1208                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
1209                 this.write_immediate(*old, dest)?; // old value is returned
1210                 Ok(())
1211             }
1212             AtomicOp::Max => {
1213                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
1214                 this.write_immediate(*old, dest)?; // old value is returned
1215                 Ok(())
1216             }
1217             AtomicOp::MirOp(op, neg) => {
1218                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
1219                 this.write_immediate(*old, dest)?; // old value is returned
1220                 Ok(())
1221             }
1222         }
1223     }
1224
1225     fn atomic_exchange(
1226         &mut self,
1227         args: &[OpTy<'tcx, Tag>],
1228         dest: &PlaceTy<'tcx, Tag>,
1229         atomic: AtomicRwOp,
1230     ) -> InterpResult<'tcx> {
1231         let this = self.eval_context_mut();
1232
1233         let [place, new] = check_arg_count(args)?;
1234         let place = this.deref_operand(place)?;
1235         let new = this.read_scalar(new)?;
1236
1237         // Check alignment requirements. Atomics must always be aligned to their size,
1238         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1239         // be 8-aligned).
1240         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1241         this.check_ptr_access_align(
1242             place.ptr,
1243             place.layout.size,
1244             align,
1245             CheckInAllocMsg::MemoryAccessTest,
1246         )?;
1247
1248         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
1249         this.write_scalar(old, dest)?; // old value is returned
1250         Ok(())
1251     }
1252
1253     fn atomic_compare_exchange_impl(
1254         &mut self,
1255         args: &[OpTy<'tcx, Tag>],
1256         dest: &PlaceTy<'tcx, Tag>,
1257         success: AtomicRwOp,
1258         fail: AtomicReadOp,
1259         can_fail_spuriously: bool,
1260     ) -> InterpResult<'tcx> {
1261         let this = self.eval_context_mut();
1262
1263         let [place, expect_old, new] = check_arg_count(args)?;
1264         let place = this.deref_operand(place)?;
1265         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
1266         let new = this.read_scalar(new)?;
1267
1268         // Check alignment requirements. Atomics must always be aligned to their size,
1269         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1270         // be 8-aligned).
1271         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1272         this.check_ptr_access_align(
1273             place.ptr,
1274             place.layout.size,
1275             align,
1276             CheckInAllocMsg::MemoryAccessTest,
1277         )?;
1278
1279         let old = this.atomic_compare_exchange_scalar(
1280             &place,
1281             &expect_old,
1282             new,
1283             success,
1284             fail,
1285             can_fail_spuriously,
1286         )?;
1287
1288         // Return old value.
1289         this.write_immediate(old, dest)?;
1290         Ok(())
1291     }
1292
1293     fn atomic_compare_exchange(
1294         &mut self,
1295         args: &[OpTy<'tcx, Tag>],
1296         dest: &PlaceTy<'tcx, Tag>,
1297         success: AtomicRwOp,
1298         fail: AtomicReadOp,
1299     ) -> InterpResult<'tcx> {
1300         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
1301     }
1302
1303     fn atomic_compare_exchange_weak(
1304         &mut self,
1305         args: &[OpTy<'tcx, Tag>],
1306         dest: &PlaceTy<'tcx, Tag>,
1307         success: AtomicRwOp,
1308         fail: AtomicReadOp,
1309     ) -> InterpResult<'tcx> {
1310         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
1311     }
1312
1313     fn float_to_int_unchecked<F>(
1314         &self,
1315         f: F,
1316         dest_ty: ty::Ty<'tcx>,
1317     ) -> InterpResult<'tcx, Scalar<Tag>>
1318     where
1319         F: Float + Into<Scalar<Tag>>,
1320     {
1321         let this = self.eval_context_ref();
1322
1323         // Step 1: cut off the fractional part of `f`. The result of this is
1324         // guaranteed to be precisely representable in IEEE floats.
1325         let f = f.round_to_integral(Round::TowardZero).value;
1326
1327         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
1328         Ok(match dest_ty.kind() {
1329             // Unsigned
1330             ty::Uint(t) => {
1331                 let size = Integer::from_uint_ty(this, *t).size();
1332                 let res = f.to_u128(size.bits_usize());
1333                 if res.status.is_empty() {
1334                     // No status flags means there was no further rounding or other loss of precision.
1335                     Scalar::from_uint(res.value, size)
1336                 } else {
1337                     // `f` was not representable in this integer type.
1338                     throw_ub_format!(
1339                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1340                         f,
1341                         dest_ty,
1342                     );
1343                 }
1344             }
1345             // Signed
1346             ty::Int(t) => {
1347                 let size = Integer::from_int_ty(this, *t).size();
1348                 let res = f.to_i128(size.bits_usize());
1349                 if res.status.is_empty() {
1350                     // No status flags means there was no further rounding or other loss of precision.
1351                     Scalar::from_int(res.value, size)
1352                 } else {
1353                     // `f` was not representable in this integer type.
1354                     throw_ub_format!(
1355                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1356                         f,
1357                         dest_ty,
1358                     );
1359                 }
1360             }
1361             // Nothing else
1362             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
1363         })
1364     }
1365 }
1366
1367 fn fmax_op<'tcx>(
1368     left: &ImmTy<'tcx, Tag>,
1369     right: &ImmTy<'tcx, Tag>,
1370 ) -> InterpResult<'tcx, Scalar<Tag>> {
1371     assert_eq!(left.layout.ty, right.layout.ty);
1372     let ty::Float(float_ty) = left.layout.ty.kind() else {
1373         bug!("fmax operand is not a float")
1374     };
1375     let left = left.to_scalar()?;
1376     let right = right.to_scalar()?;
1377     Ok(match float_ty {
1378         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
1379         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
1380     })
1381 }
1382
1383 fn fmin_op<'tcx>(
1384     left: &ImmTy<'tcx, Tag>,
1385     right: &ImmTy<'tcx, Tag>,
1386 ) -> InterpResult<'tcx, Scalar<Tag>> {
1387     assert_eq!(left.layout.ty, right.layout.ty);
1388     let ty::Float(float_ty) = left.layout.ty.kind() else {
1389         bug!("fmin operand is not a float")
1390     };
1391     let left = left.to_scalar()?;
1392     let right = right.to_scalar()?;
1393     Ok(match float_ty {
1394         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
1395         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
1396     })
1397 }
1398
1399 fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
1400     // SIMD uses all-1 as pattern for "true"
1401     let val = if b { -1 } else { 0 };
1402     Scalar::from_int(val, size)
1403 }
1404
1405 fn simd_element_to_bool(elem: ImmTy<'_, Tag>) -> InterpResult<'_, bool> {
1406     let val = elem.to_scalar()?.to_int(elem.layout.size)?;
1407     Ok(match val {
1408         0 => false,
1409         -1 => true,
1410         _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
1411     })
1412 }
1413
1414 fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 {
1415     assert!(idx < vec_len);
1416     match endianess {
1417         Endian::Little => idx,
1418         Endian::Big => vec_len - 1 - idx, // reverse order of bits
1419     }
1420 }