X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=src%2Fintrinsics%2Fmod.rs;h=313b62c5770b6fe823fa6e6723f73d20bab6d4e8;hb=0134088cd2eed9a48fee853d52b44b019a46cb2f;hp=3563aa250a9b8fb7c8fd1c6af0e89479495196ec;hpb=d4048407880212bb820f85d2e06e5fc262cce897;p=rust.git diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 3563aa250a9..313b62c5770 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -8,22 +8,25 @@ pub(crate) use cpuid::codegen_cpuid_call; pub(crate) use llvm::codegen_llvm_intrinsic_call; -use crate::prelude::*; use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_span::symbol::{kw, sym}; + +use crate::prelude::*; +use cranelift_codegen::ir::AtomicRmwOp; macro intrinsic_pat { (_) => { _ }, ($name:ident) => { - stringify!($name) + sym::$name + }, + (kw.$name:ident) => { + kw::$name }, ($name:literal) => { - stringify!($name) + $name }, - ($x:ident . $($xs:tt).*) => { - concat!(stringify!($x), ".", intrinsic_pat!($($xs).*)) - } } macro intrinsic_arg { @@ -86,7 +89,7 @@ )*) => { match $intrinsic { $( - stringify!($name) => { + sym::$name => { assert!($substs.is_noop()); if let [$(ref $arg),*] = *$args { let ($($arg,)*) = ( @@ -112,46 +115,14 @@ } } -macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) { - crate::atomic_shim::lock_global_lock($fx); - - let clif_ty = $fx.clif_type($T).unwrap(); - let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0); - let new = $fx.bcx.ins().$op(old, $src); - $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0); - $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T))); - - crate::atomic_shim::unlock_global_lock($fx); -} - -macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) { - crate::atomic_shim::lock_global_lock($fx); - - // Read old - let clif_ty = $fx.clif_type($T).unwrap(); - let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0); - - // Compare - let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src); - let new = $fx.bcx.ins().select(is_eq, old, $src); - - // Write new - $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0); - - let ret_val = CValue::by_val(old, $ret.layout()); - $ret.write_cvalue($fx, ret_val); - - crate::atomic_shim::unlock_global_lock($fx); -} - macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) { match $ty.kind() { - ty::Uint(_) | ty::Int(_) => {} + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} _ => { $fx.tcx.sess.span_err( $span, &format!( - "`{}` intrinsic: expected basic integer type, found `{:?}`", + "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`", $intrinsic, $ty ), ); @@ -171,30 +142,9 @@ } } -fn lane_type_and_count<'tcx>( - tcx: TyCtxt<'tcx>, - layout: TyAndLayout<'tcx>, -) -> (TyAndLayout<'tcx>, u16) { - assert!(layout.ty.is_simd()); - let lane_count = match layout.fields { - rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(), - _ => unreachable!("lane_type_and_count({:?})", layout), - }; - let lane_layout = layout - .field( - &ty::layout::LayoutCx { - tcx, - param_env: ParamEnv::reveal_all(), - }, - 0, - ) - .unwrap(); - (lane_layout, lane_count) -} - pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option { - let (element, count) = match &layout.abi { - Abi::Vector { element, count } => (element.clone(), *count), + let (element, count) = match layout.abi { + Abi::Vector { element, count } => (element, count), _ => unreachable!(), }; @@ -205,12 +155,12 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx } } -fn simd_for_each_lane<'tcx, M: Module>( - fx: &mut FunctionCx<'_, 'tcx, M>, +fn simd_for_each_lane<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, val: CValue<'tcx>, ret: CPlace<'tcx>, f: impl Fn( - &mut FunctionCx<'_, 'tcx, M>, + &mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, TyAndLayout<'tcx>, Value, @@ -218,27 +168,28 @@ fn simd_for_each_lane<'tcx, M: Module>( ) { let layout = val.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); for lane_idx in 0..lane_count { - let lane_idx = mir::Field::new(lane_idx.try_into().unwrap()); - let lane = val.value_field(fx, lane_idx).load_scalar(fx); + let lane = val.value_lane(fx, lane_idx).load_scalar(fx); let res_lane = f(fx, lane_layout, ret_lane_layout, lane); - ret.place_field(fx, lane_idx).write_cvalue(fx, res_lane); + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); } } -fn simd_pair_for_each_lane<'tcx, M: Module>( - fx: &mut FunctionCx<'_, 'tcx, M>, +fn simd_pair_for_each_lane<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, x: CValue<'tcx>, y: CValue<'tcx>, ret: CPlace<'tcx>, f: impl Fn( - &mut FunctionCx<'_, 'tcx, M>, + &mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, TyAndLayout<'tcx>, Value, @@ -248,65 +199,71 @@ fn simd_pair_for_each_lane<'tcx, M: Module>( assert_eq!(x.layout(), y.layout()); let layout = x.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); - for lane in 0..lane_count { - let lane = mir::Field::new(lane.try_into().unwrap()); - let x_lane = x.value_field(fx, lane).load_scalar(fx); - let y_lane = y.value_field(fx, lane).load_scalar(fx); + for lane_idx in 0..lane_count { + let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx); + let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx); let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane); - ret.place_field(fx, lane).write_cvalue(fx, res_lane); + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); } } -fn simd_reduce<'tcx, M: Module>( - fx: &mut FunctionCx<'_, 'tcx, M>, +fn simd_reduce<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, val: CValue<'tcx>, + acc: Option, ret: CPlace<'tcx>, - f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value, + f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value, ) { - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); assert_eq!(lane_layout, ret.layout()); - let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); - for lane_idx in 1..lane_count { - let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) - .load_scalar(fx); + let (mut res_val, start_lane) = + if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) }; + for lane_idx in start_lane..lane_count { + let lane = val.value_lane(fx, lane_idx).load_scalar(fx); res_val = f(fx, lane_layout, res_val, lane); } let res = CValue::by_val(res_val, lane_layout); ret.write_cvalue(fx, res); } -fn simd_reduce_bool<'tcx, M: Module>( - fx: &mut FunctionCx<'_, 'tcx, M>, +// FIXME move all uses to `simd_reduce` +fn simd_reduce_bool<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, val: CValue<'tcx>, ret: CPlace<'tcx>, - f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value, + f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value, ) { - let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); assert!(ret.layout().ty.is_bool()); - let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); + let res_val = val.value_lane(fx, 0).load_scalar(fx); let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean for lane_idx in 1..lane_count { - let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) - .load_scalar(fx); + let lane = val.value_lane(fx, lane_idx).load_scalar(fx); let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean res_val = f(fx, res_val, lane); } + let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 { + fx.bcx.ins().ireduce(types::I8, res_val) + } else { + res_val + }; let res = CValue::by_val(res_val, ret.layout()); ret.write_cvalue(fx, res); } fn bool_to_zero_or_max_uint<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, + fx: &mut FunctionCx<'_, '_, 'tcx>, layout: TyAndLayout<'tcx>, val: Value, ) -> CValue<'tcx> { @@ -335,7 +292,11 @@ fn bool_to_zero_or_max_uint<'tcx>( if let Some(vector_ty) = vector_ty { let x = $x.load_scalar($fx); let y = $y.load_scalar($fx); - let val = $fx.bcx.ins().icmp(IntCC::$cc, x, y); + let val = if vector_ty.lane_type().is_float() { + $fx.bcx.ins().fcmp(FloatCC::$cc_f, x, y) + } else { + $fx.bcx.ins().icmp(IntCC::$cc, x, y) + }; // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1. let val = $fx.bcx.ins().raw_bitcast(vector_ty, val); @@ -440,30 +401,24 @@ fn bool_to_zero_or_max_uint<'tcx>( } pub(crate) fn codegen_intrinsic_call<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, + fx: &mut FunctionCx<'_, '_, 'tcx>, instance: Instance<'tcx>, args: &[mir::Operand<'tcx>], destination: Option<(CPlace<'tcx>, BasicBlock)>, span: Span, ) { - let def_id = instance.def_id(); + let intrinsic = fx.tcx.item_name(instance.def_id()); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; - let ret = match destination { Some((place, _)) => place, None => { // Insert non returning intrinsics here match intrinsic { - "abort" => { + sym::abort => { trap_abort(fx, "Called intrinsic::abort."); } - "unreachable" => { - trap_unreachable(fx, "[corruption] Called intrinsic::unreachable."); - } - "transmute" => { + sym::transmute => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } _ => unimplemented!("unsupported instrinsic {}", intrinsic), @@ -472,7 +427,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( } }; - if intrinsic.starts_with("simd_") { + if intrinsic.as_str().starts_with("simd_") { self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span); let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1); fx.bcx.ins().jump(ret_block, &[]); @@ -522,8 +477,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( sinf64(flt) -> f64 => sin, cosf32(flt) -> f32 => cosf, cosf64(flt) -> f64 => cos, - tanf32(flt) -> f32 => tanf, - tanf64(flt) -> f64 => tan, } intrinsic_match! { @@ -548,12 +501,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( count }; - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::copy_nonoverlapping { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; // NOTE: the volatile variants have src and dst swapped @@ -567,20 +520,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::volatile_copy_nonoverlapping_memory { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; - discriminant_value, (c ptr) { - let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout); - let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout()); - ret.write_cvalue(fx, discr); - }; size_of_val, (c ptr) { let layout = fx.layout_of(T); let size = if layout.is_unsized() { @@ -610,27 +557,28 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); }; - _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) { + unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem + | unchecked_shl | unchecked_shr, (c x, c y) { // FIXME trap on overflow let bin_op = match intrinsic { - "unchecked_add" => BinOp::Add, - "unchecked_sub" => BinOp::Sub, - "unchecked_div" | "exact_div" => BinOp::Div, - "unchecked_rem" => BinOp::Rem, - "unchecked_shl" => BinOp::Shl, - "unchecked_shr" => BinOp::Shr, - _ => unreachable!("intrinsic {}", intrinsic), + sym::unchecked_add => BinOp::Add, + sym::unchecked_sub => BinOp::Sub, + sym::unchecked_div | sym::exact_div => BinOp::Div, + sym::unchecked_rem => BinOp::Rem, + sym::unchecked_shl => BinOp::Shl, + sym::unchecked_shr => BinOp::Shr, + _ => unreachable!(), }; let res = crate::num::codegen_int_binop(fx, bin_op, x, y); ret.write_cvalue(fx, res); }; - _ if intrinsic.ends_with("_with_overflow"), (c x, c y) { + add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) { assert_eq!(x.layout().ty, y.layout().ty); let bin_op = match intrinsic { - "add_with_overflow" => BinOp::Add, - "sub_with_overflow" => BinOp::Sub, - "mul_with_overflow" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), + sym::add_with_overflow => BinOp::Add, + sym::sub_with_overflow => BinOp::Sub, + sym::mul_with_overflow => BinOp::Mul, + _ => unreachable!(), }; let res = crate::num::codegen_checked_int_binop( @@ -641,28 +589,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("wrapping_"), (c x, c y) { - assert_eq!(x.layout().ty, y.layout().ty); - let bin_op = match intrinsic { - "wrapping_add" => BinOp::Add, - "wrapping_sub" => BinOp::Sub, - "wrapping_mul" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), - }; - let res = crate::num::codegen_int_binop( - fx, - bin_op, - x, - y, - ); - ret.write_cvalue(fx, res); - }; - _ if intrinsic.starts_with("saturating_"), (c lhs, c rhs) { + saturating_add | saturating_sub, (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { - "saturating_add" => BinOp::Add, - "saturating_sub" => BinOp::Sub, - _ => unreachable!("intrinsic {}", intrinsic), + sym::saturating_add => BinOp::Add, + sym::saturating_sub => BinOp::Sub, + _ => unreachable!(), }; let signed = type_sign(T); @@ -677,21 +609,18 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let (val, has_overflow) = checked_res.load_scalar_pair(fx); let clif_ty = fx.clif_type(T).unwrap(); - // `select.i8` is not implemented by Cranelift. - let has_overflow = fx.bcx.ins().uextend(types::I32, has_overflow); - let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); let val = match (intrinsic, signed) { - ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val), - ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val), - ("saturating_add", true) => { + (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val), + (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val), + (sym::saturating_add, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); fx.bcx.ins().select(has_overflow, sat_val, val) } - ("saturating_sub", true) => { + (sym::saturating_sub, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); @@ -744,39 +673,17 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let dst_ptr = dst.load_scalar(fx); // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset - fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count); + fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count); }; ctlz | ctlz_nonzero, (v arg) { // FIXME trap on `ctlz_nonzero` with zero arg. - let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 { - // FIXME verify this algorithm is correct - let (lsb, msb) = fx.bcx.ins().isplit(arg); - let lsb_lz = fx.bcx.ins().clz(lsb); - let msb_lz = fx.bcx.ins().clz(msb); - let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0); - let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64); - let res = fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz); - fx.bcx.ins().uextend(types::I128, res) - } else { - fx.bcx.ins().clz(arg) - }; + let res = fx.bcx.ins().clz(arg); let res = CValue::by_val(res, fx.layout_of(T)); ret.write_cvalue(fx, res); }; cttz | cttz_nonzero, (v arg) { // FIXME trap on `cttz_nonzero` with zero arg. - let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 { - // FIXME verify this algorithm is correct - let (lsb, msb) = fx.bcx.ins().isplit(arg); - let lsb_tz = fx.bcx.ins().ctz(lsb); - let msb_tz = fx.bcx.ins().ctz(msb); - let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0); - let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64); - let res = fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz); - fx.bcx.ins().uextend(types::I128, res) - } else { - fx.bcx.ins().ctz(arg) - }; + let res = fx.bcx.ins().ctz(arg); let res = CValue::by_val(res, fx.layout_of(T)); ret.write_cvalue(fx, res); }; @@ -865,7 +772,7 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { } ty => unreachable!("bswap {}", ty), } - }; + } let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T)); ret.write_cvalue(fx, res); }; @@ -880,7 +787,7 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { return; } - if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { + if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true) { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to zero-initialize type `{}`, which is invalid", T), @@ -889,7 +796,7 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { return; } - if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { + if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false) { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to leave type `{}` uninitialized, which is invalid", T), @@ -901,7 +808,6 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { volatile_load | unaligned_volatile_load, (c ptr) { // Cranelift treats loads as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_load let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); @@ -910,13 +816,12 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { }; volatile_store | unaligned_volatile_store, (v ptr, c val) { // Cranelift treats stores as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_store let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); }; - size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { + pref_align_of | needs_drop | type_id | type_name | variant_count, () { let const_val = fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap(); let val = crate::constant::codegen_const_value( @@ -952,161 +857,214 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { ret.write_cvalue(fx, caller_location); }; - _ if intrinsic.starts_with("atomic_fence"), () { - crate::atomic_shim::lock_global_lock(fx); - crate::atomic_shim::unlock_global_lock(fx); + _ if intrinsic.as_str().starts_with("atomic_fence"), () { + fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_singlethreadfence"), () { - crate::atomic_shim::lock_global_lock(fx); - crate::atomic_shim::unlock_global_lock(fx); + _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () { + // FIXME use a compiler fence once Cranelift supports it + fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_load"), (c ptr) { - crate::atomic_shim::lock_global_lock(fx); + _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) { + validate_atomic_type!(fx, intrinsic, span, T); + let ty = fx.clif_type(T).unwrap(); - let inner_layout = - fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - validate_atomic_type!(fx, intrinsic, span, inner_layout.ty); - let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); - ret.write_cvalue(fx, val); + let val = fx.bcx.ins().atomic_load(ty, MemFlags::trusted(), ptr); - crate::atomic_shim::unlock_global_lock(fx); + let val = CValue::by_val(val, fx.layout_of(T)); + ret.write_cvalue(fx, val); }; - _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) { + _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) { validate_atomic_type!(fx, intrinsic, span, val.layout().ty); - crate::atomic_shim::lock_global_lock(fx); + let val = val.load_scalar(fx); - let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); - dest.write_cvalue(fx, val); - - crate::atomic_shim::unlock_global_lock(fx); + fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); }; - _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, T); - - crate::atomic_shim::lock_global_lock(fx); + _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) { + let layout = new.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); - // Read old - let clif_ty = fx.clif_type(T).unwrap(); - let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0); - ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T))); + let new = new.load_scalar(fx); - // Write new - let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout()); - dest.write_cvalue(fx, src); + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new); - crate::atomic_shim::unlock_global_lock(fx); + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* - validate_atomic_type!(fx, intrinsic, span, T); + _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* + let layout = new.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); let test_old = test_old.load_scalar(fx); let new = new.load_scalar(fx); - crate::atomic_shim::lock_global_lock(fx); - - // Read old - let clif_ty = fx.clif_type(T).unwrap(); - let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0); - - // Compare + let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new); let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old); - let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old - - // Write new - fx.bcx.ins().store(MemFlags::new(), new, ptr, 0); let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout()); - ret.write_cvalue(fx, ret_val); - - crate::atomic_shim::unlock_global_lock(fx); + ret.write_cvalue(fx, ret_val) }; - _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) { + let layout = amount.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let amount = amount.load_scalar(fx); - atomic_binop_return_old! (fx, iadd(ptr, amount) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) { + let layout = amount.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let amount = amount.load_scalar(fx); - atomic_binop_return_old! (fx, isub(ptr, amount) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let src = src.load_scalar(fx); - atomic_binop_return_old! (fx, band(ptr, src) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, T); + _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); let src = src.load_scalar(fx); - crate::atomic_shim::lock_global_lock(fx); + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src); - let clif_ty = fx.clif_type(T).unwrap(); - let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0); - let and = fx.bcx.ins().band(old, src); - let new = fx.bcx.ins().bnot(and); - fx.bcx.ins().store(MemFlags::new(), new, ptr, 0); - ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T))); - - crate::atomic_shim::unlock_global_lock(fx); + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let src = src.load_scalar(fx); - atomic_binop_return_old! (fx, bor(ptr, src) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let src = src.load_scalar(fx); - atomic_binop_return_old! (fx, bxor(ptr, src) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; + _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); - _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); let src = src.load_scalar(fx); - atomic_minmax!(fx, IntCC::SignedGreaterThan, (ptr, src) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let src = src.load_scalar(fx); - atomic_minmax!(fx, IntCC::UnsignedGreaterThan, (ptr, src) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let src = src.load_scalar(fx); - atomic_minmax!(fx, IntCC::SignedLessThan, (ptr, src) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) { - validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) { + let layout = src.layout(); + validate_atomic_type!(fx, intrinsic, span, layout.ty); + let ty = fx.clif_type(layout.ty).unwrap(); + let src = src.load_scalar(fx); - atomic_minmax!(fx, IntCC::UnsignedLessThan, (ptr, src) -> ret); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); }; + // In Rust floating point min and max don't propagate NaN. In Cranelift they do however. + // For this reason it is necessary to use `a.is_nan() ? b : (a >= b ? b : a)` for `minnumf*` + // and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing + // a float against itself. Only in case of NaN is it not equal to itself. minnumf32, (v a, v b) { - let val = fx.bcx.ins().fmin(a, b); + let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); + let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b); + let temp = fx.bcx.ins().select(a_ge_b, b, a); + let val = fx.bcx.ins().select(a_is_nan, b, temp); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); ret.write_cvalue(fx, val); }; minnumf64, (v a, v b) { - let val = fx.bcx.ins().fmin(a, b); + let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); + let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b); + let temp = fx.bcx.ins().select(a_ge_b, b, a); + let val = fx.bcx.ins().select(a_is_nan, b, temp); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); }; maxnumf32, (v a, v b) { - let val = fx.bcx.ins().fmax(a, b); + let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); + let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b); + let temp = fx.bcx.ins().select(a_le_b, b, a); + let val = fx.bcx.ins().select(a_is_nan, b, temp); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); ret.write_cvalue(fx, val); }; maxnumf64, (v a, v b) { - let val = fx.bcx.ins().fmax(a, b); + let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); + let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b); + let temp = fx.bcx.ins().select(a_le_b, b, a); + let val = fx.bcx.ins().select(a_is_nan, b, temp); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); }; - try, (v f, v data, v _catch_fn) { + kw.Try, (v f, v data, v _catch_fn) { // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: CallConv::triple_default(fx.triple()), @@ -1123,11 +1081,11 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) { let res = crate::num::codegen_float_binop(fx, match intrinsic { - "fadd_fast" => BinOp::Add, - "fsub_fast" => BinOp::Sub, - "fmul_fast" => BinOp::Mul, - "fdiv_fast" => BinOp::Div, - "frem_fast" => BinOp::Rem, + sym::fadd_fast => BinOp::Add, + sym::fsub_fast => BinOp::Sub, + sym::fmul_fast => BinOp::Mul, + sym::fdiv_fast => BinOp::Div, + sym::frem_fast => BinOp::Rem, _ => unreachable!(), }, x, y); ret.write_cvalue(fx, res); @@ -1142,6 +1100,45 @@ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { ); ret.write_cvalue(fx, CValue::by_val(res, ret.layout())); }; + + raw_eq, (v lhs_ref, v rhs_ref) { + fn type_by_size(size: Size) -> Option { + Type::int(size.bits().try_into().ok()?) + } + + let size = fx.layout_of(T).layout.size; + // FIXME add and use emit_small_memcmp + let is_eq_value = + if size == Size::ZERO { + // No bytes means they're trivially equal + fx.bcx.ins().iconst(types::I8, 1) + } else if let Some(clty) = type_by_size(size) { + // Can't use `trusted` for these loads; they could be unaligned. + let mut flags = MemFlags::new(); + flags.set_notrap(); + let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0); + let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0); + let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val); + fx.bcx.ins().bint(types::I8, eq) + } else { + // Just call `memcmp` (like slices do in core) when the + // size is too large or it's not a power-of-two. + let signed_bytes = i64::try_from(size.bytes()).unwrap(); + let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes); + let params = vec![AbiParam::new(fx.pointer_type); 3]; + let returns = vec![AbiParam::new(types::I32)]; + let args = &[lhs_ref, rhs_ref, bytes_val]; + let cmp = fx.lib_call("memcmp", params, returns, args)[0]; + let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0); + fx.bcx.ins().bint(types::I8, eq) + }; + ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout())); + }; + + black_box, (c a) { + // FIXME implement black_box semantics + ret.write_cvalue(fx, a); + }; } if let Some((_, dest)) = destination {