src/shims/intrinsics/mod.rs

   1 mod atomic;
   2 mod simd;
   3
   4 use std::iter;
   5
   6 use log::trace;
   7
   8 use rustc_apfloat::{Float, Round};
   9 use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
  10 use rustc_middle::{mir, ty, ty::FloatTy};
  11 use rustc_target::abi::Integer;
  12
  13 use crate::*;
  14 use atomic::EvalContextExt as _;
  15 use helpers::check_arg_count;
  16 use simd::EvalContextExt as _;
  17
  18 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  19 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  20     fn call_intrinsic(
  21         &mut self,
  22         instance: ty::Instance<'tcx>,
  23         args: &[OpTy<'tcx, Provenance>],
  24         dest: &PlaceTy<'tcx, Provenance>,
  25         ret: Option<mir::BasicBlock>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         // See if the core engine can handle this intrinsic.
  31         if this.emulate_intrinsic(instance, args, dest, ret)? {
  32             return Ok(());
  33         }
  34
  35         // All remaining supported intrinsics have a return place.
  36         let intrinsic_name = this.tcx.item_name(instance.def_id());
  37         let intrinsic_name = intrinsic_name.as_str();
  38         let ret = match ret {
  39             None => throw_unsup_format!("unimplemented (diverging) intrinsic: `{intrinsic_name}`"),
  40             Some(p) => p,
  41         };
  42
  43         // Some intrinsics are special and need the "ret".
  44         match intrinsic_name {
  45             "try" => return this.handle_try(args, dest, ret),
  46             _ => {}
  47         }
  48
  49         // The rest jumps to `ret` immediately.
  50         this.emulate_intrinsic_by_name(intrinsic_name, args, dest)?;
  51
  52         trace!("{:?}", this.dump_place(**dest));
  53         this.go_to_block(ret);
  54         Ok(())
  55     }
  56
  57     /// Emulates a Miri-supported intrinsic (not supported by the core engine).
  58     fn emulate_intrinsic_by_name(
  59         &mut self,
  60         intrinsic_name: &str,
  61         args: &[OpTy<'tcx, Provenance>],
  62         dest: &PlaceTy<'tcx, Provenance>,
  63     ) -> InterpResult<'tcx> {
  64         let this = self.eval_context_mut();
  65
  66         if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
  67             return this.emulate_atomic_intrinsic(name, args, dest);
  68         }
  69         if let Some(name) = intrinsic_name.strip_prefix("simd_") {
  70             return this.emulate_simd_intrinsic(name, args, dest);
  71         }
  72
  73         match intrinsic_name {
  74             // Miri overwriting CTFE intrinsics.
  75             "ptr_guaranteed_eq" => {
  76                 let [left, right] = check_arg_count(args)?;
  77                 let left = this.read_immediate(left)?;
  78                 let right = this.read_immediate(right)?;
  79                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  80             }
  81             "ptr_guaranteed_ne" => {
  82                 let [left, right] = check_arg_count(args)?;
  83                 let left = this.read_immediate(left)?;
  84                 let right = this.read_immediate(right)?;
  85                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  86             }
  87             "const_allocate" => {
  88                 // For now, for compatibility with the run-time implementation of this, we just return null.
  89                 // See <https://github.com/rust-lang/rust/issues/93935>.
  90                 this.write_null(dest)?;
  91             }
  92             "const_deallocate" => {
  93                 // complete NOP
  94             }
  95
  96             // Raw memory accesses
  97             "volatile_load" => {
  98                 let [place] = check_arg_count(args)?;
  99                 let place = this.deref_operand(place)?;
 100                 this.copy_op(&place.into(), dest, /*allow_transmute*/ false)?;
 101             }
 102             "volatile_store" => {
 103                 let [place, dest] = check_arg_count(args)?;
 104                 let place = this.deref_operand(place)?;
 105                 this.copy_op(dest, &place.into(), /*allow_transmute*/ false)?;
 106             }
 107
 108             "write_bytes" | "volatile_set_memory" => {
 109                 let [ptr, val_byte, count] = check_arg_count(args)?;
 110                 let ty = ptr.layout.ty.builtin_deref(true).unwrap().ty;
 111                 let ty_layout = this.layout_of(ty)?;
 112                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
 113                 let ptr = this.read_pointer(ptr)?;
 114                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
 115                 // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
 116                 // but no actual allocation can be big enough for the difference to be noticeable.
 117                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
 118                     err_ub_format!("overflow computing total size of `{intrinsic_name}`")
 119                 })?;
 120                 this.write_bytes_ptr(ptr, iter::repeat(val_byte).take(byte_count.bytes_usize()))?;
 121             }
 122
 123             // Floating-point operations
 124             "fabsf32" => {
 125                 let [f] = check_arg_count(args)?;
 126                 let f = this.read_scalar(f)?.to_f32()?;
 127                 // Can be implemented in soft-floats.
 128                 this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
 129             }
 130             "fabsf64" => {
 131                 let [f] = check_arg_count(args)?;
 132                 let f = this.read_scalar(f)?.to_f64()?;
 133                 // Can be implemented in soft-floats.
 134                 this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
 135             }
 136             #[rustfmt::skip]
 137             | "sinf32"
 138             | "cosf32"
 139             | "sqrtf32"
 140             | "expf32"
 141             | "exp2f32"
 142             | "logf32"
 143             | "log10f32"
 144             | "log2f32"
 145             | "floorf32"
 146             | "ceilf32"
 147             | "truncf32"
 148             | "roundf32"
 149             => {
 150                 let [f] = check_arg_count(args)?;
 151                 // FIXME: Using host floats.
 152                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 153                 let f = match intrinsic_name {
 154                     "sinf32" => f.sin(),
 155                     "cosf32" => f.cos(),
 156                     "sqrtf32" => f.sqrt(),
 157                     "expf32" => f.exp(),
 158                     "exp2f32" => f.exp2(),
 159                     "logf32" => f.ln(),
 160                     "log10f32" => f.log10(),
 161                     "log2f32" => f.log2(),
 162                     "floorf32" => f.floor(),
 163                     "ceilf32" => f.ceil(),
 164                     "truncf32" => f.trunc(),
 165                     "roundf32" => f.round(),
 166                     _ => bug!(),
 167                 };
 168                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 169             }
 170
 171             #[rustfmt::skip]
 172             | "sinf64"
 173             | "cosf64"
 174             | "sqrtf64"
 175             | "expf64"
 176             | "exp2f64"
 177             | "logf64"
 178             | "log10f64"
 179             | "log2f64"
 180             | "floorf64"
 181             | "ceilf64"
 182             | "truncf64"
 183             | "roundf64"
 184             => {
 185                 let [f] = check_arg_count(args)?;
 186                 // FIXME: Using host floats.
 187                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 188                 let f = match intrinsic_name {
 189                     "sinf64" => f.sin(),
 190                     "cosf64" => f.cos(),
 191                     "sqrtf64" => f.sqrt(),
 192                     "expf64" => f.exp(),
 193                     "exp2f64" => f.exp2(),
 194                     "logf64" => f.ln(),
 195                     "log10f64" => f.log10(),
 196                     "log2f64" => f.log2(),
 197                     "floorf64" => f.floor(),
 198                     "ceilf64" => f.ceil(),
 199                     "truncf64" => f.trunc(),
 200                     "roundf64" => f.round(),
 201                     _ => bug!(),
 202                 };
 203                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 204             }
 205
 206             #[rustfmt::skip]
 207             | "fadd_fast"
 208             | "fsub_fast"
 209             | "fmul_fast"
 210             | "fdiv_fast"
 211             | "frem_fast"
 212             => {
 213                 let [a, b] = check_arg_count(args)?;
 214                 let a = this.read_immediate(a)?;
 215                 let b = this.read_immediate(b)?;
 216                 let op = match intrinsic_name {
 217                     "fadd_fast" => mir::BinOp::Add,
 218                     "fsub_fast" => mir::BinOp::Sub,
 219                     "fmul_fast" => mir::BinOp::Mul,
 220                     "fdiv_fast" => mir::BinOp::Div,
 221                     "frem_fast" => mir::BinOp::Rem,
 222                     _ => bug!(),
 223                 };
 224                 let float_finite = |x: &ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 225                     Ok(match x.layout.ty.kind() {
 226                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 227                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 228                         _ => bug!(
 229                             "`{intrinsic_name}` called with non-float input type {ty:?}",
 230                             ty = x.layout.ty,
 231                         ),
 232                     })
 233                 };
 234                 match (float_finite(&a)?, float_finite(&b)?) {
 235                     (false, false) => throw_ub_format!(
 236                         "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
 237                     ),
 238                     (false, _) => throw_ub_format!(
 239                         "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
 240                     ),
 241                     (_, false) => throw_ub_format!(
 242                         "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
 243                     ),
 244                     _ => {}
 245                 }
 246                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 247             }
 248
 249             #[rustfmt::skip]
 250             | "minnumf32"
 251             | "maxnumf32"
 252             | "copysignf32"
 253             => {
 254                 let [a, b] = check_arg_count(args)?;
 255                 let a = this.read_scalar(a)?.to_f32()?;
 256                 let b = this.read_scalar(b)?.to_f32()?;
 257                 let res = match intrinsic_name {
 258                     "minnumf32" => a.min(b),
 259                     "maxnumf32" => a.max(b),
 260                     "copysignf32" => a.copy_sign(b),
 261                     _ => bug!(),
 262                 };
 263                 this.write_scalar(Scalar::from_f32(res), dest)?;
 264             }
 265
 266             #[rustfmt::skip]
 267             | "minnumf64"
 268             | "maxnumf64"
 269             | "copysignf64"
 270             => {
 271                 let [a, b] = check_arg_count(args)?;
 272                 let a = this.read_scalar(a)?.to_f64()?;
 273                 let b = this.read_scalar(b)?.to_f64()?;
 274                 let res = match intrinsic_name {
 275                     "minnumf64" => a.min(b),
 276                     "maxnumf64" => a.max(b),
 277                     "copysignf64" => a.copy_sign(b),
 278                     _ => bug!(),
 279                 };
 280                 this.write_scalar(Scalar::from_f64(res), dest)?;
 281             }
 282
 283             "powf32" => {
 284                 let [f, f2] = check_arg_count(args)?;
 285                 // FIXME: Using host floats.
 286                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 287                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 288                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 289             }
 290
 291             "powf64" => {
 292                 let [f, f2] = check_arg_count(args)?;
 293                 // FIXME: Using host floats.
 294                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 295                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 296                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 297             }
 298
 299             "fmaf32" => {
 300                 let [a, b, c] = check_arg_count(args)?;
 301                 let a = this.read_scalar(a)?.to_f32()?;
 302                 let b = this.read_scalar(b)?.to_f32()?;
 303                 let c = this.read_scalar(c)?.to_f32()?;
 304                 let res = a.mul_add(b, c).value;
 305                 this.write_scalar(Scalar::from_f32(res), dest)?;
 306             }
 307
 308             "fmaf64" => {
 309                 let [a, b, c] = check_arg_count(args)?;
 310                 let a = this.read_scalar(a)?.to_f64()?;
 311                 let b = this.read_scalar(b)?.to_f64()?;
 312                 let c = this.read_scalar(c)?.to_f64()?;
 313                 let res = a.mul_add(b, c).value;
 314                 this.write_scalar(Scalar::from_f64(res), dest)?;
 315             }
 316
 317             "powif32" => {
 318                 let [f, i] = check_arg_count(args)?;
 319                 // FIXME: Using host floats.
 320                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 321                 let i = this.read_scalar(i)?.to_i32()?;
 322                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 323             }
 324
 325             "powif64" => {
 326                 let [f, i] = check_arg_count(args)?;
 327                 // FIXME: Using host floats.
 328                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 329                 let i = this.read_scalar(i)?.to_i32()?;
 330                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 331             }
 332
 333             "float_to_int_unchecked" => {
 334                 let [val] = check_arg_count(args)?;
 335                 let val = this.read_immediate(val)?;
 336
 337                 let res = match val.layout.ty.kind() {
 338                     ty::Float(FloatTy::F32) =>
 339                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 340                     ty::Float(FloatTy::F64) =>
 341                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 342                     _ =>
 343                         span_bug!(
 344                             this.cur_span(),
 345                             "`float_to_int_unchecked` called with non-float input type {:?}",
 346                             val.layout.ty
 347                         ),
 348                 };
 349
 350                 this.write_scalar(res, dest)?;
 351             }
 352
 353             // Other
 354             "exact_div" => {
 355                 let [num, denom] = check_arg_count(args)?;
 356                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 357             }
 358
 359             "breakpoint" => {
 360                 let [] = check_arg_count(args)?;
 361                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 362                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 363             }
 364
 365             name => throw_unsup_format!("unimplemented intrinsic: `{name}`"),
 366         }
 367
 368         Ok(())
 369     }
 370
 371     fn float_to_int_unchecked<F>(
 372         &self,
 373         f: F,
 374         dest_ty: ty::Ty<'tcx>,
 375     ) -> InterpResult<'tcx, Scalar<Provenance>>
 376     where
 377         F: Float + Into<Scalar<Provenance>>,
 378     {
 379         let this = self.eval_context_ref();
 380
 381         // Step 1: cut off the fractional part of `f`. The result of this is
 382         // guaranteed to be precisely representable in IEEE floats.
 383         let f = f.round_to_integral(Round::TowardZero).value;
 384
 385         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 386         Ok(match dest_ty.kind() {
 387             // Unsigned
 388             ty::Uint(t) => {
 389                 let size = Integer::from_uint_ty(this, *t).size();
 390                 let res = f.to_u128(size.bits_usize());
 391                 if res.status.is_empty() {
 392                     // No status flags means there was no further rounding or other loss of precision.
 393                     Scalar::from_uint(res.value, size)
 394                 } else {
 395                     // `f` was not representable in this integer type.
 396                     throw_ub_format!(
 397                         "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
 398                     );
 399                 }
 400             }
 401             // Signed
 402             ty::Int(t) => {
 403                 let size = Integer::from_int_ty(this, *t).size();
 404                 let res = f.to_i128(size.bits_usize());
 405                 if res.status.is_empty() {
 406                     // No status flags means there was no further rounding or other loss of precision.
 407                     Scalar::from_int(res.value, size)
 408                 } else {
 409                     // `f` was not representable in this integer type.
 410                     throw_ub_format!(
 411                         "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
 412                     );
 413                 }
 414             }
 415             // Nothing else
 416             _ =>
 417                 span_bug!(
 418                     this.cur_span(),
 419                     "`float_to_int_unchecked` called with non-int output type {dest_ty:?}"
 420                 ),
 421         })
 422     }
 423 }