src/shims/intrinsics/mod.rs

   1 mod atomic;
   2 mod simd;
   3
   4 use std::iter;
   5
   6 use log::trace;
   7
   8 use rustc_apfloat::{Float, Round};
   9 use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
  10 use rustc_middle::{mir, ty, ty::FloatTy};
  11 use rustc_target::abi::Integer;
  12
  13 use crate::*;
  14 use atomic::EvalContextExt as _;
  15 use helpers::check_arg_count;
  16 use simd::EvalContextExt as _;
  17
  18 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  19 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  20     fn call_intrinsic(
  21         &mut self,
  22         instance: ty::Instance<'tcx>,
  23         args: &[OpTy<'tcx, Tag>],
  24         dest: &PlaceTy<'tcx, Tag>,
  25         ret: Option<mir::BasicBlock>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         // See if the core engine can handle this intrinsic.
  31         if this.emulate_intrinsic(instance, args, dest, ret)? {
  32             return Ok(());
  33         }
  34
  35         // All remaining supported intrinsics have a return place.
  36         let intrinsic_name = this.tcx.item_name(instance.def_id());
  37         let intrinsic_name = intrinsic_name.as_str();
  38         let ret = match ret {
  39             None => throw_unsup_format!("unimplemented (diverging) intrinsic: `{intrinsic_name}`"),
  40             Some(p) => p,
  41         };
  42
  43         // Some intrinsics are special and need the "ret".
  44         match intrinsic_name {
  45             "try" => return this.handle_try(args, dest, ret),
  46             _ => {}
  47         }
  48
  49         // The rest jumps to `ret` immediately.
  50         this.emulate_intrinsic_by_name(intrinsic_name, args, dest)?;
  51
  52         trace!("{:?}", this.dump_place(**dest));
  53         this.go_to_block(ret);
  54         Ok(())
  55     }
  56
  57     /// Emulates a Miri-supported intrinsic (not supported by the core engine).
  58     fn emulate_intrinsic_by_name(
  59         &mut self,
  60         intrinsic_name: &str,
  61         args: &[OpTy<'tcx, Tag>],
  62         dest: &PlaceTy<'tcx, Tag>,
  63     ) -> InterpResult<'tcx> {
  64         let this = self.eval_context_mut();
  65
  66         if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
  67             return this.emulate_atomic_intrinsic(name, args, dest);
  68         }
  69         if let Some(name) = intrinsic_name.strip_prefix("simd_") {
  70             return this.emulate_simd_intrinsic(name, args, dest);
  71         }
  72
  73         match intrinsic_name {
  74             // Miri overwriting CTFE intrinsics.
  75             "ptr_guaranteed_eq" => {
  76                 let [left, right] = check_arg_count(args)?;
  77                 let left = this.read_immediate(left)?;
  78                 let right = this.read_immediate(right)?;
  79                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  80             }
  81             "ptr_guaranteed_ne" => {
  82                 let [left, right] = check_arg_count(args)?;
  83                 let left = this.read_immediate(left)?;
  84                 let right = this.read_immediate(right)?;
  85                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  86             }
  87             "const_allocate" => {
  88                 // For now, for compatibility with the run-time implementation of this, we just return null.
  89                 // See <https://github.com/rust-lang/rust/issues/93935>.
  90                 this.write_null(dest)?;
  91             }
  92             "const_deallocate" => {
  93                 // complete NOP
  94             }
  95
  96             // Raw memory accesses
  97             "volatile_load" => {
  98                 let [place] = check_arg_count(args)?;
  99                 let place = this.deref_operand(place)?;
 100                 this.copy_op(&place.into(), dest, /*allow_transmute*/ false)?;
 101             }
 102             "volatile_store" => {
 103                 let [place, dest] = check_arg_count(args)?;
 104                 let place = this.deref_operand(place)?;
 105                 this.copy_op(dest, &place.into(), /*allow_transmute*/ false)?;
 106             }
 107
 108             "write_bytes" | "volatile_set_memory" => {
 109                 let [ptr, val_byte, count] = check_arg_count(args)?;
 110                 let ty = ptr.layout.ty.builtin_deref(true).unwrap().ty;
 111                 let ty_layout = this.layout_of(ty)?;
 112                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
 113                 let ptr = this.read_pointer(ptr)?;
 114                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
 115                 // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
 116                 // but no actual allocation can be big enough for the difference to be noticeable.
 117                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
 118                     err_ub_format!("overflow computing total size of `{intrinsic_name}`")
 119                 })?;
 120                 this.write_bytes_ptr(
 121                     ptr,
 122                     iter::repeat(val_byte).take(byte_count.bytes() as usize),
 123                 )?;
 124             }
 125
 126             // Floating-point operations
 127             "fabsf32" => {
 128                 let [f] = check_arg_count(args)?;
 129                 let f = this.read_scalar(f)?.to_f32()?;
 130                 // Can be implemented in soft-floats.
 131                 this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
 132             }
 133             "fabsf64" => {
 134                 let [f] = check_arg_count(args)?;
 135                 let f = this.read_scalar(f)?.to_f64()?;
 136                 // Can be implemented in soft-floats.
 137                 this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
 138             }
 139             #[rustfmt::skip]
 140             | "sinf32"
 141             | "cosf32"
 142             | "sqrtf32"
 143             | "expf32"
 144             | "exp2f32"
 145             | "logf32"
 146             | "log10f32"
 147             | "log2f32"
 148             | "floorf32"
 149             | "ceilf32"
 150             | "truncf32"
 151             | "roundf32"
 152             => {
 153                 let [f] = check_arg_count(args)?;
 154                 // FIXME: Using host floats.
 155                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 156                 let f = match intrinsic_name {
 157                     "sinf32" => f.sin(),
 158                     "cosf32" => f.cos(),
 159                     "sqrtf32" => f.sqrt(),
 160                     "expf32" => f.exp(),
 161                     "exp2f32" => f.exp2(),
 162                     "logf32" => f.ln(),
 163                     "log10f32" => f.log10(),
 164                     "log2f32" => f.log2(),
 165                     "floorf32" => f.floor(),
 166                     "ceilf32" => f.ceil(),
 167                     "truncf32" => f.trunc(),
 168                     "roundf32" => f.round(),
 169                     _ => bug!(),
 170                 };
 171                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 172             }
 173
 174             #[rustfmt::skip]
 175             | "sinf64"
 176             | "cosf64"
 177             | "sqrtf64"
 178             | "expf64"
 179             | "exp2f64"
 180             | "logf64"
 181             | "log10f64"
 182             | "log2f64"
 183             | "floorf64"
 184             | "ceilf64"
 185             | "truncf64"
 186             | "roundf64"
 187             => {
 188                 let [f] = check_arg_count(args)?;
 189                 // FIXME: Using host floats.
 190                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 191                 let f = match intrinsic_name {
 192                     "sinf64" => f.sin(),
 193                     "cosf64" => f.cos(),
 194                     "sqrtf64" => f.sqrt(),
 195                     "expf64" => f.exp(),
 196                     "exp2f64" => f.exp2(),
 197                     "logf64" => f.ln(),
 198                     "log10f64" => f.log10(),
 199                     "log2f64" => f.log2(),
 200                     "floorf64" => f.floor(),
 201                     "ceilf64" => f.ceil(),
 202                     "truncf64" => f.trunc(),
 203                     "roundf64" => f.round(),
 204                     _ => bug!(),
 205                 };
 206                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 207             }
 208
 209             #[rustfmt::skip]
 210             | "fadd_fast"
 211             | "fsub_fast"
 212             | "fmul_fast"
 213             | "fdiv_fast"
 214             | "frem_fast"
 215             => {
 216                 let [a, b] = check_arg_count(args)?;
 217                 let a = this.read_immediate(a)?;
 218                 let b = this.read_immediate(b)?;
 219                 let op = match intrinsic_name {
 220                     "fadd_fast" => mir::BinOp::Add,
 221                     "fsub_fast" => mir::BinOp::Sub,
 222                     "fmul_fast" => mir::BinOp::Mul,
 223                     "fdiv_fast" => mir::BinOp::Div,
 224                     "frem_fast" => mir::BinOp::Rem,
 225                     _ => bug!(),
 226                 };
 227                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 228                     Ok(match x.layout.ty.kind() {
 229                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 230                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 231                         _ => bug!(
 232                             "`{intrinsic_name}` called with non-float input type {ty:?}",
 233                             ty = x.layout.ty,
 234                         ),
 235                     })
 236                 };
 237                 match (float_finite(a)?, float_finite(b)?) {
 238                     (false, false) => throw_ub_format!(
 239                         "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
 240                     ),
 241                     (false, _) => throw_ub_format!(
 242                         "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
 243                     ),
 244                     (_, false) => throw_ub_format!(
 245                         "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
 246                     ),
 247                     _ => {}
 248                 }
 249                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 250             }
 251
 252             #[rustfmt::skip]
 253             | "minnumf32"
 254             | "maxnumf32"
 255             | "copysignf32"
 256             => {
 257                 let [a, b] = check_arg_count(args)?;
 258                 let a = this.read_scalar(a)?.to_f32()?;
 259                 let b = this.read_scalar(b)?.to_f32()?;
 260                 let res = match intrinsic_name {
 261                     "minnumf32" => a.min(b),
 262                     "maxnumf32" => a.max(b),
 263                     "copysignf32" => a.copy_sign(b),
 264                     _ => bug!(),
 265                 };
 266                 this.write_scalar(Scalar::from_f32(res), dest)?;
 267             }
 268
 269             #[rustfmt::skip]
 270             | "minnumf64"
 271             | "maxnumf64"
 272             | "copysignf64"
 273             => {
 274                 let [a, b] = check_arg_count(args)?;
 275                 let a = this.read_scalar(a)?.to_f64()?;
 276                 let b = this.read_scalar(b)?.to_f64()?;
 277                 let res = match intrinsic_name {
 278                     "minnumf64" => a.min(b),
 279                     "maxnumf64" => a.max(b),
 280                     "copysignf64" => a.copy_sign(b),
 281                     _ => bug!(),
 282                 };
 283                 this.write_scalar(Scalar::from_f64(res), dest)?;
 284             }
 285
 286             "powf32" => {
 287                 let [f, f2] = check_arg_count(args)?;
 288                 // FIXME: Using host floats.
 289                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 290                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 291                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 292             }
 293
 294             "powf64" => {
 295                 let [f, f2] = check_arg_count(args)?;
 296                 // FIXME: Using host floats.
 297                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 298                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 299                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 300             }
 301
 302             "fmaf32" => {
 303                 let [a, b, c] = check_arg_count(args)?;
 304                 let a = this.read_scalar(a)?.to_f32()?;
 305                 let b = this.read_scalar(b)?.to_f32()?;
 306                 let c = this.read_scalar(c)?.to_f32()?;
 307                 let res = a.mul_add(b, c).value;
 308                 this.write_scalar(Scalar::from_f32(res), dest)?;
 309             }
 310
 311             "fmaf64" => {
 312                 let [a, b, c] = check_arg_count(args)?;
 313                 let a = this.read_scalar(a)?.to_f64()?;
 314                 let b = this.read_scalar(b)?.to_f64()?;
 315                 let c = this.read_scalar(c)?.to_f64()?;
 316                 let res = a.mul_add(b, c).value;
 317                 this.write_scalar(Scalar::from_f64(res), dest)?;
 318             }
 319
 320             "powif32" => {
 321                 let [f, i] = check_arg_count(args)?;
 322                 // FIXME: Using host floats.
 323                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 324                 let i = this.read_scalar(i)?.to_i32()?;
 325                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 326             }
 327
 328             "powif64" => {
 329                 let [f, i] = check_arg_count(args)?;
 330                 // FIXME: Using host floats.
 331                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 332                 let i = this.read_scalar(i)?.to_i32()?;
 333                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 334             }
 335
 336             "float_to_int_unchecked" => {
 337                 let [val] = check_arg_count(args)?;
 338                 let val = this.read_immediate(val)?;
 339
 340                 let res = match val.layout.ty.kind() {
 341                     ty::Float(FloatTy::F32) =>
 342                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 343                     ty::Float(FloatTy::F64) =>
 344                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 345                     _ =>
 346                         span_bug!(
 347                             this.cur_span(),
 348                             "`float_to_int_unchecked` called with non-float input type {:?}",
 349                             val.layout.ty
 350                         ),
 351                 };
 352
 353                 this.write_scalar(res, dest)?;
 354             }
 355
 356             // Other
 357             "exact_div" => {
 358                 let [num, denom] = check_arg_count(args)?;
 359                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 360             }
 361
 362             "breakpoint" => {
 363                 let [] = check_arg_count(args)?;
 364                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 365                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 366             }
 367
 368             name => throw_unsup_format!("unimplemented intrinsic: `{name}`"),
 369         }
 370
 371         Ok(())
 372     }
 373
 374     fn float_to_int_unchecked<F>(
 375         &self,
 376         f: F,
 377         dest_ty: ty::Ty<'tcx>,
 378     ) -> InterpResult<'tcx, Scalar<Tag>>
 379     where
 380         F: Float + Into<Scalar<Tag>>,
 381     {
 382         let this = self.eval_context_ref();
 383
 384         // Step 1: cut off the fractional part of `f`. The result of this is
 385         // guaranteed to be precisely representable in IEEE floats.
 386         let f = f.round_to_integral(Round::TowardZero).value;
 387
 388         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 389         Ok(match dest_ty.kind() {
 390             // Unsigned
 391             ty::Uint(t) => {
 392                 let size = Integer::from_uint_ty(this, *t).size();
 393                 let res = f.to_u128(size.bits_usize());
 394                 if res.status.is_empty() {
 395                     // No status flags means there was no further rounding or other loss of precision.
 396                     Scalar::from_uint(res.value, size)
 397                 } else {
 398                     // `f` was not representable in this integer type.
 399                     throw_ub_format!(
 400                         "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
 401                     );
 402                 }
 403             }
 404             // Signed
 405             ty::Int(t) => {
 406                 let size = Integer::from_int_ty(this, *t).size();
 407                 let res = f.to_i128(size.bits_usize());
 408                 if res.status.is_empty() {
 409                     // No status flags means there was no further rounding or other loss of precision.
 410                     Scalar::from_int(res.value, size)
 411                 } else {
 412                     // `f` was not representable in this integer type.
 413                     throw_ub_format!(
 414                         "`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
 415                     );
 416                 }
 417             }
 418             // Nothing else
 419             _ =>
 420                 span_bug!(
 421                     this.cur_span(),
 422                     "`float_to_int_unchecked` called with non-int output type {dest_ty:?}"
 423                 ),
 424         })
 425     }
 426 }