-use crate::llvm;
-use crate::llvm_util;
use crate::abi::{Abi, FnAbi, LlvmType, PassMode};
+use crate::builder::Builder;
use crate::context::CodegenCx;
+use crate::llvm;
+use crate::llvm_util;
use crate::type_::Type;
use crate::type_of::LayoutLlvmExt;
-use crate::builder::Builder;
-use crate::value::Value;
use crate::va_arg::emit_va_arg;
-use rustc_codegen_ssa::MemFlags;
-use rustc_codegen_ssa::mir::place::PlaceRef;
-use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
-use rustc_codegen_ssa::glue;
-use rustc_codegen_ssa::base::{to_immediate, wants_msvc_seh, compare_simd_types};
+use crate::value::Value;
+use rustc::ty::layout::{self, FnAbiExt, HasTyCtxt, LayoutOf, Primitive};
use rustc::ty::{self, Ty};
-use rustc::ty::layout::{self, FnAbiExt, LayoutOf, HasTyCtxt, Primitive};
-use rustc::mir::interpret::GlobalId;
+use rustc::{bug, span_bug};
+use rustc_codegen_ssa::base::{compare_simd_types, to_immediate, wants_msvc_seh};
use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
-use rustc::hir;
+use rustc_codegen_ssa::glue;
+use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
+use rustc_codegen_ssa::mir::place::PlaceRef;
+use rustc_codegen_ssa::MemFlags;
+use rustc_hir as hir;
use rustc_target::abi::HasDataLayout;
use syntax::ast;
-use rustc::{bug, span_bug};
use rustc_codegen_ssa::common::span_invalid_monomorphization_error;
use rustc_codegen_ssa::traits::*;
-use syntax_pos::Span;
+use rustc_span::Span;
use std::cmp::Ordering;
-use std::{iter, i128, u128};
+use std::{i128, iter, u128};
fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: &str) -> Option<&'ll Value> {
let llvm_name = match name {
"roundf64" => "llvm.round.f64",
"assume" => "llvm.assume",
"abort" => "llvm.trap",
- _ => return None
+ _ => return None,
};
Some(cx.get_intrinsic(&llvm_name))
}
span: Span,
) {
let tcx = self.tcx;
- let callee_ty = instance.ty(tcx);
+ let callee_ty = instance.monomorphic_ty(tcx);
let (def_id, substs) = match callee_ty.kind {
ty::FnDef(def_id, substs) => (def_id, substs),
- _ => bug!("expected fn item type, found {}", callee_ty)
+ _ => bug!("expected fn item type, found {}", callee_ty),
};
let sig = callee_ty.fn_sig(tcx);
let simple = get_simple_intrinsic(self, name);
let llval = match name {
- _ if simple.is_some() => {
- self.call(simple.unwrap(),
- &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
- None)
- }
+ _ if simple.is_some() => self.call(
+ simple.unwrap(),
+ &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
+ None,
+ ),
"unreachable" => {
return;
- },
+ }
"likely" => {
let expect = self.get_intrinsic(&("llvm.expect.i1"));
self.call(expect, &[args[0].immediate(), self.const_bool(true)], None)
self.call(expect, &[args[0].immediate(), self.const_bool(false)], None)
}
"try" => {
- try_intrinsic(self,
- args[0].immediate(),
- args[1].immediate(),
- args[2].immediate(),
- llresult);
+ try_intrinsic(
+ self,
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ llresult,
+ );
return;
}
"breakpoint" => {
let llfn = self.get_intrinsic(&("llvm.debugtrap"));
self.call(llfn, &[], None)
}
- "va_start" => {
- self.va_start(args[0].immediate())
- }
- "va_end" => {
- self.va_end(args[0].immediate())
- }
+ "va_start" => self.va_start(args[0].immediate()),
+ "va_end" => self.va_end(args[0].immediate()),
"va_copy" => {
let intrinsic = self.cx().get_intrinsic(&("llvm.va_copy"));
self.call(intrinsic, &[args[0].immediate(), args[1].immediate()], None)
// less than 4 bytes in length. If it is, promote
// the integer to a `i32` and truncate the result
// back to the smaller type.
- let promoted_result = emit_va_arg(self, args[0],
- tcx.types.i32);
+ let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
self.trunc(promoted_result, llret_ty)
} else {
emit_va_arg(self, args[0], ret_ty)
}
}
- Primitive::F64 |
- Primitive::Pointer => {
+ Primitive::F64 | Primitive::Pointer => {
emit_va_arg(self, args[0], ret_ty)
}
// `va_arg` should never be used with the return type f32.
- Primitive::F32 => {
- bug!("the va_arg intrinsic does not work with `f32`")
- }
+ Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"),
}
}
- _ => {
- bug!("the va_arg intrinsic does not work with non-scalar types")
- }
+ _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
}
}
"size_of_val" => {
self.const_usize(self.align_of(tp_ty).bytes())
}
}
- "size_of" |
- "pref_align_of" |
- "min_align_of" |
- "needs_drop" |
- "type_id" |
- "type_name" => {
- let gid = GlobalId {
- instance,
- promoted: None,
- };
- let ty_name = self.tcx.const_eval(ty::ParamEnv::reveal_all().and(gid)).unwrap();
+ "size_of" | "pref_align_of" | "min_align_of" | "needs_drop" | "type_id"
+ | "type_name" => {
+ let ty_name = self
+ .tcx
+ .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None)
+ .unwrap();
OperandRef::from_const(self, ty_name).immediate_or_packed_pair(self)
}
"init" => {
ty,
llresult,
self.const_u8(0),
- self.const_usize(1)
+ self.const_usize(1),
);
}
return;
}
"copy_nonoverlapping" => {
- copy_intrinsic(self, false, false, substs.type_at(0),
- args[1].immediate(), args[0].immediate(), args[2].immediate());
+ copy_intrinsic(
+ self,
+ false,
+ false,
+ substs.type_at(0),
+ args[1].immediate(),
+ args[0].immediate(),
+ args[2].immediate(),
+ );
return;
}
"copy" => {
- copy_intrinsic(self, true, false, substs.type_at(0),
- args[1].immediate(), args[0].immediate(), args[2].immediate());
+ copy_intrinsic(
+ self,
+ true,
+ false,
+ substs.type_at(0),
+ args[1].immediate(),
+ args[0].immediate(),
+ args[2].immediate(),
+ );
return;
}
"write_bytes" => {
- memset_intrinsic(self, false, substs.type_at(0),
- args[0].immediate(), args[1].immediate(), args[2].immediate());
+ memset_intrinsic(
+ self,
+ false,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
return;
}
"volatile_copy_nonoverlapping_memory" => {
- copy_intrinsic(self, false, true, substs.type_at(0),
- args[0].immediate(), args[1].immediate(), args[2].immediate());
+ copy_intrinsic(
+ self,
+ false,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
return;
}
"volatile_copy_memory" => {
- copy_intrinsic(self, true, true, substs.type_at(0),
- args[0].immediate(), args[1].immediate(), args[2].immediate());
+ copy_intrinsic(
+ self,
+ true,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
return;
}
"volatile_set_memory" => {
- memset_intrinsic(self, true, substs.type_at(0),
- args[0].immediate(), args[1].immediate(), args[2].immediate());
+ memset_intrinsic(
+ self,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
return;
}
"volatile_load" | "unaligned_volatile_load" => {
llvm::LLVMSetAlignment(load, align);
}
to_immediate(self, load, self.layout_of(tp_ty))
- },
+ }
"volatile_store" => {
let dst = args[0].deref(self.cx());
args[1].val.volatile_store(self, dst);
return;
- },
+ }
"unaligned_volatile_store" => {
let dst = args[0].deref(self.cx());
args[1].val.unaligned_volatile_store(self, dst);
return;
- },
- "prefetch_read_data" | "prefetch_write_data" |
- "prefetch_read_instruction" | "prefetch_write_instruction" => {
+ }
+ "prefetch_read_data"
+ | "prefetch_write_data"
+ | "prefetch_read_instruction"
+ | "prefetch_write_instruction" => {
let expect = self.get_intrinsic(&("llvm.prefetch"));
let (rw, cache_type) = match name {
"prefetch_read_data" => (0, 1),
"prefetch_write_data" => (1, 1),
"prefetch_read_instruction" => (0, 0),
"prefetch_write_instruction" => (1, 0),
- _ => bug!()
+ _ => bug!(),
};
- self.call(expect, &[
- args[0].immediate(),
- self.const_i32(rw),
- args[1].immediate(),
- self.const_i32(cache_type)
- ], None)
- },
- "ctlz" | "ctlz_nonzero" | "cttz" | "cttz_nonzero" | "ctpop" | "bswap" |
- "bitreverse" | "add_with_overflow" | "sub_with_overflow" |
- "mul_with_overflow" | "wrapping_add" | "wrapping_sub" | "wrapping_mul" |
- "unchecked_div" | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" |
- "unchecked_add" | "unchecked_sub" | "unchecked_mul" | "exact_div" |
- "rotate_left" | "rotate_right" | "saturating_add" | "saturating_sub" => {
+ self.call(
+ expect,
+ &[
+ args[0].immediate(),
+ self.const_i32(rw),
+ args[1].immediate(),
+ self.const_i32(cache_type),
+ ],
+ None,
+ )
+ }
+ "ctlz" | "ctlz_nonzero" | "cttz" | "cttz_nonzero" | "ctpop" | "bswap"
+ | "bitreverse" | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow"
+ | "wrapping_add" | "wrapping_sub" | "wrapping_mul" | "unchecked_div"
+ | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" | "unchecked_add"
+ | "unchecked_sub" | "unchecked_mul" | "exact_div" | "rotate_left" | "rotate_right"
+ | "saturating_add" | "saturating_sub" => {
let ty = arg_tys[0];
match int_type_width_signed(ty, self) {
- Some((width, signed)) =>
- match name {
- "ctlz" | "cttz" => {
- let y = self.const_bool(false);
- let llfn = self.get_intrinsic(
- &format!("llvm.{}.i{}", name, width),
- );
- self.call(llfn, &[args[0].immediate(), y], None)
- }
- "ctlz_nonzero" | "cttz_nonzero" => {
- let y = self.const_bool(true);
- let llvm_name = &format!("llvm.{}.i{}", &name[..4], width);
- let llfn = self.get_intrinsic(llvm_name);
- self.call(llfn, &[args[0].immediate(), y], None)
- }
- "ctpop" => self.call(
- self.get_intrinsic(&format!("llvm.ctpop.i{}", width)),
- &[args[0].immediate()],
- None
- ),
- "bswap" => {
- if width == 8 {
- args[0].immediate() // byte swap a u8/i8 is just a no-op
- } else {
- self.call(
- self.get_intrinsic(
- &format!("llvm.bswap.i{}", width),
- ),
- &[args[0].immediate()],
- None,
- )
- }
- }
- "bitreverse" => {
+ Some((width, signed)) => match name {
+ "ctlz" | "cttz" => {
+ let y = self.const_bool(false);
+ let llfn = self.get_intrinsic(&format!("llvm.{}.i{}", name, width));
+ self.call(llfn, &[args[0].immediate(), y], None)
+ }
+ "ctlz_nonzero" | "cttz_nonzero" => {
+ let y = self.const_bool(true);
+ let llvm_name = &format!("llvm.{}.i{}", &name[..4], width);
+ let llfn = self.get_intrinsic(llvm_name);
+ self.call(llfn, &[args[0].immediate(), y], None)
+ }
+ "ctpop" => self.call(
+ self.get_intrinsic(&format!("llvm.ctpop.i{}", width)),
+ &[args[0].immediate()],
+ None,
+ ),
+ "bswap" => {
+ if width == 8 {
+ args[0].immediate() // byte swap a u8/i8 is just a no-op
+ } else {
self.call(
- self.get_intrinsic(
- &format!("llvm.bitreverse.i{}", width),
- ),
+ self.get_intrinsic(&format!("llvm.bswap.i{}", width)),
&[args[0].immediate()],
None,
)
}
- "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => {
- let intrinsic = format!("llvm.{}{}.with.overflow.i{}",
- if signed { 's' } else { 'u' },
- &name[..3], width);
- let llfn = self.get_intrinsic(&intrinsic);
-
- // Convert `i1` to a `bool`, and write it to the out parameter
- let pair = self.call(llfn, &[
- args[0].immediate(),
- args[1].immediate()
- ], None);
+ }
+ "bitreverse" => self.call(
+ self.get_intrinsic(&format!("llvm.bitreverse.i{}", width)),
+ &[args[0].immediate()],
+ None,
+ ),
+ "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => {
+ let intrinsic = format!(
+ "llvm.{}{}.with.overflow.i{}",
+ if signed { 's' } else { 'u' },
+ &name[..3],
+ width
+ );
+ let llfn = self.get_intrinsic(&intrinsic);
+
+ // Convert `i1` to a `bool`, and write it to the out parameter
+ let pair =
+ self.call(llfn, &[args[0].immediate(), args[1].immediate()], None);
+ let val = self.extract_value(pair, 0);
+ let overflow = self.extract_value(pair, 1);
+ let overflow = self.zext(overflow, self.type_bool());
+
+ let dest = result.project_field(self, 0);
+ self.store(val, dest.llval, dest.align);
+ let dest = result.project_field(self, 1);
+ self.store(overflow, dest.llval, dest.align);
+
+ return;
+ }
+ "wrapping_add" => self.add(args[0].immediate(), args[1].immediate()),
+ "wrapping_sub" => self.sub(args[0].immediate(), args[1].immediate()),
+ "wrapping_mul" => self.mul(args[0].immediate(), args[1].immediate()),
+ "exact_div" => {
+ if signed {
+ self.exactsdiv(args[0].immediate(), args[1].immediate())
+ } else {
+ self.exactudiv(args[0].immediate(), args[1].immediate())
+ }
+ }
+ "unchecked_div" => {
+ if signed {
+ self.sdiv(args[0].immediate(), args[1].immediate())
+ } else {
+ self.udiv(args[0].immediate(), args[1].immediate())
+ }
+ }
+ "unchecked_rem" => {
+ if signed {
+ self.srem(args[0].immediate(), args[1].immediate())
+ } else {
+ self.urem(args[0].immediate(), args[1].immediate())
+ }
+ }
+ "unchecked_shl" => self.shl(args[0].immediate(), args[1].immediate()),
+ "unchecked_shr" => {
+ if signed {
+ self.ashr(args[0].immediate(), args[1].immediate())
+ } else {
+ self.lshr(args[0].immediate(), args[1].immediate())
+ }
+ }
+ "unchecked_add" => {
+ if signed {
+ self.unchecked_sadd(args[0].immediate(), args[1].immediate())
+ } else {
+ self.unchecked_uadd(args[0].immediate(), args[1].immediate())
+ }
+ }
+ "unchecked_sub" => {
+ if signed {
+ self.unchecked_ssub(args[0].immediate(), args[1].immediate())
+ } else {
+ self.unchecked_usub(args[0].immediate(), args[1].immediate())
+ }
+ }
+ "unchecked_mul" => {
+ if signed {
+ self.unchecked_smul(args[0].immediate(), args[1].immediate())
+ } else {
+ self.unchecked_umul(args[0].immediate(), args[1].immediate())
+ }
+ }
+ "rotate_left" | "rotate_right" => {
+ let is_left = name == "rotate_left";
+ let val = args[0].immediate();
+ let raw_shift = args[1].immediate();
+ // rotate = funnel shift with first two args the same
+ let llvm_name =
+ &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
+ let llfn = self.get_intrinsic(llvm_name);
+ self.call(llfn, &[val, val, raw_shift], None)
+ }
+ "saturating_add" | "saturating_sub" => {
+ let is_add = name == "saturating_add";
+ let lhs = args[0].immediate();
+ let rhs = args[1].immediate();
+ if llvm_util::get_major_version() >= 8 {
+ let llvm_name = &format!(
+ "llvm.{}{}.sat.i{}",
+ if signed { 's' } else { 'u' },
+ if is_add { "add" } else { "sub" },
+ width
+ );
+ let llfn = self.get_intrinsic(llvm_name);
+ self.call(llfn, &[lhs, rhs], None)
+ } else {
+ let llvm_name = &format!(
+ "llvm.{}{}.with.overflow.i{}",
+ if signed { 's' } else { 'u' },
+ if is_add { "add" } else { "sub" },
+ width
+ );
+ let llfn = self.get_intrinsic(llvm_name);
+ let pair = self.call(llfn, &[lhs, rhs], None);
let val = self.extract_value(pair, 0);
let overflow = self.extract_value(pair, 1);
- let overflow = self.zext(overflow, self.type_bool());
-
- let dest = result.project_field(self, 0);
- self.store(val, dest.llval, dest.align);
- let dest = result.project_field(self, 1);
- self.store(overflow, dest.llval, dest.align);
-
- return;
- },
- "wrapping_add" => self.add(args[0].immediate(), args[1].immediate()),
- "wrapping_sub" => self.sub(args[0].immediate(), args[1].immediate()),
- "wrapping_mul" => self.mul(args[0].immediate(), args[1].immediate()),
- "exact_div" =>
- if signed {
- self.exactsdiv(args[0].immediate(), args[1].immediate())
+ let llty = self.type_ix(width);
+
+ let limit = if signed {
+ let limit_lo = self
+ .const_uint_big(llty, (i128::MIN >> (128 - width)) as u128);
+ let limit_hi = self
+ .const_uint_big(llty, (i128::MAX >> (128 - width)) as u128);
+ let neg = self.icmp(
+ IntPredicate::IntSLT,
+ val,
+ self.const_uint(llty, 0),
+ );
+ self.select(neg, limit_hi, limit_lo)
+ } else if is_add {
+ self.const_uint_big(llty, u128::MAX >> (128 - width))
} else {
- self.exactudiv(args[0].immediate(), args[1].immediate())
- },
- "unchecked_div" =>
- if signed {
- self.sdiv(args[0].immediate(), args[1].immediate())
- } else {
- self.udiv(args[0].immediate(), args[1].immediate())
- },
- "unchecked_rem" =>
- if signed {
- self.srem(args[0].immediate(), args[1].immediate())
- } else {
- self.urem(args[0].immediate(), args[1].immediate())
- },
- "unchecked_shl" => self.shl(args[0].immediate(), args[1].immediate()),
- "unchecked_shr" =>
- if signed {
- self.ashr(args[0].immediate(), args[1].immediate())
- } else {
- self.lshr(args[0].immediate(), args[1].immediate())
- },
- "unchecked_add" => {
- if signed {
- self.unchecked_sadd(args[0].immediate(), args[1].immediate())
- } else {
- self.unchecked_uadd(args[0].immediate(), args[1].immediate())
- }
- },
- "unchecked_sub" => {
- if signed {
- self.unchecked_ssub(args[0].immediate(), args[1].immediate())
- } else {
- self.unchecked_usub(args[0].immediate(), args[1].immediate())
- }
- },
- "unchecked_mul" => {
- if signed {
- self.unchecked_smul(args[0].immediate(), args[1].immediate())
- } else {
- self.unchecked_umul(args[0].immediate(), args[1].immediate())
- }
- },
- "rotate_left" | "rotate_right" => {
- let is_left = name == "rotate_left";
- let val = args[0].immediate();
- let raw_shift = args[1].immediate();
- // rotate = funnel shift with first two args the same
- let llvm_name = &format!("llvm.fsh{}.i{}",
- if is_left { 'l' } else { 'r' }, width);
- let llfn = self.get_intrinsic(llvm_name);
- self.call(llfn, &[val, val, raw_shift], None)
- },
- "saturating_add" | "saturating_sub" => {
- let is_add = name == "saturating_add";
- let lhs = args[0].immediate();
- let rhs = args[1].immediate();
- if llvm_util::get_major_version() >= 8 {
- let llvm_name = &format!("llvm.{}{}.sat.i{}",
- if signed { 's' } else { 'u' },
- if is_add { "add" } else { "sub" },
- width);
- let llfn = self.get_intrinsic(llvm_name);
- self.call(llfn, &[lhs, rhs], None)
- } else {
- let llvm_name = &format!("llvm.{}{}.with.overflow.i{}",
- if signed { 's' } else { 'u' },
- if is_add { "add" } else { "sub" },
- width);
- let llfn = self.get_intrinsic(llvm_name);
- let pair = self.call(llfn, &[lhs, rhs], None);
- let val = self.extract_value(pair, 0);
- let overflow = self.extract_value(pair, 1);
- let llty = self.type_ix(width);
-
- let limit = if signed {
- let limit_lo = self.const_uint_big(
- llty, (i128::MIN >> (128 - width)) as u128);
- let limit_hi = self.const_uint_big(
- llty, (i128::MAX >> (128 - width)) as u128);
- let neg = self.icmp(
- IntPredicate::IntSLT, val, self.const_uint(llty, 0));
- self.select(neg, limit_hi, limit_lo)
- } else if is_add {
- self.const_uint_big(llty, u128::MAX >> (128 - width))
- } else {
- self.const_uint(llty, 0)
- };
- self.select(overflow, limit, val)
- }
- },
- _ => bug!(),
- },
+ self.const_uint(llty, 0)
+ };
+ self.select(overflow, limit, val)
+ }
+ }
+ _ => bug!(),
+ },
None => {
span_invalid_monomorphization_error(
- tcx.sess, span,
- &format!("invalid monomorphization of `{}` intrinsic: \
- expected basic integer type, found `{}`", name, ty));
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic integer type, found `{}`",
+ name, ty
+ ),
+ );
return;
}
}
-
- },
+ }
"fadd_fast" | "fsub_fast" | "fmul_fast" | "fdiv_fast" | "frem_fast" => {
match float_type_width(arg_tys[0]) {
- Some(_width) =>
- match name {
- "fadd_fast" => self.fadd_fast(args[0].immediate(), args[1].immediate()),
- "fsub_fast" => self.fsub_fast(args[0].immediate(), args[1].immediate()),
- "fmul_fast" => self.fmul_fast(args[0].immediate(), args[1].immediate()),
- "fdiv_fast" => self.fdiv_fast(args[0].immediate(), args[1].immediate()),
- "frem_fast" => self.frem_fast(args[0].immediate(), args[1].immediate()),
- _ => bug!(),
- },
+ Some(_width) => match name {
+ "fadd_fast" => self.fadd_fast(args[0].immediate(), args[1].immediate()),
+ "fsub_fast" => self.fsub_fast(args[0].immediate(), args[1].immediate()),
+ "fmul_fast" => self.fmul_fast(args[0].immediate(), args[1].immediate()),
+ "fdiv_fast" => self.fdiv_fast(args[0].immediate(), args[1].immediate()),
+ "frem_fast" => self.frem_fast(args[0].immediate(), args[1].immediate()),
+ _ => bug!(),
+ },
None => {
span_invalid_monomorphization_error(
- tcx.sess, span,
- &format!("invalid monomorphization of `{}` intrinsic: \
- expected basic float type, found `{}`", name, arg_tys[0]));
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic float type, found `{}`",
+ name, arg_tys[0]
+ ),
+ );
return;
}
}
- },
+ }
"float_to_int_approx_unchecked" => {
if float_type_width(arg_tys[0]).is_none() {
span_invalid_monomorphization_error(
- tcx.sess, span,
- &format!("invalid monomorphization of `float_to_int_approx_unchecked` \
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `float_to_int_approx_unchecked` \
intrinsic: expected basic float type, \
- found `{}`", arg_tys[0]));
+ found `{}`",
+ arg_tys[0]
+ ),
+ );
return;
}
match int_type_width_signed(ret_ty, self.cx) {
}
None => {
span_invalid_monomorphization_error(
- tcx.sess, span,
- &format!("invalid monomorphization of `float_to_int_approx_unchecked` \
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `float_to_int_approx_unchecked` \
intrinsic: expected basic integer type, \
- found `{}`", ret_ty));
+ found `{}`",
+ ret_ty
+ ),
+ );
return;
}
}
}
- "discriminant_value" => {
- args[0].deref(self.cx()).codegen_get_discr(self, ret_ty)
- }
+ "discriminant_value" => args[0].deref(self.cx()).codegen_get_discr(self, ret_ty),
name if name.starts_with("simd_") => {
- match generic_simd_intrinsic(self, name,
- callee_ty,
- args,
- ret_ty, llret_ty,
- span) {
+ match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
Ok(llval) => llval,
- Err(()) => return
+ Err(()) => return,
}
}
// This requires that atomic intrinsics follow a specific naming pattern:
// "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
name if name.starts_with("atomic_") => {
use rustc_codegen_ssa::common::AtomicOrdering::*;
- use rustc_codegen_ssa::common::
- {SynchronizationScope, AtomicRmwBinOp};
+ use rustc_codegen_ssa::common::{AtomicRmwBinOp, SynchronizationScope};
let split: Vec<&str> = name.split('_').collect();
3 => match split[2] {
"unordered" => (Unordered, Unordered),
"relaxed" => (Monotonic, Monotonic),
- "acq" => (Acquire, Acquire),
- "rel" => (Release, Monotonic),
- "acqrel" => (AcquireRelease, Acquire),
- "failrelaxed" if is_cxchg =>
- (SequentiallyConsistent, Monotonic),
- "failacq" if is_cxchg =>
- (SequentiallyConsistent, Acquire),
- _ => self.sess().fatal("unknown ordering in atomic intrinsic")
+ "acq" => (Acquire, Acquire),
+ "rel" => (Release, Monotonic),
+ "acqrel" => (AcquireRelease, Acquire),
+ "failrelaxed" if is_cxchg => (SequentiallyConsistent, Monotonic),
+ "failacq" if is_cxchg => (SequentiallyConsistent, Acquire),
+ _ => self.sess().fatal("unknown ordering in atomic intrinsic"),
},
4 => match (split[2], split[3]) {
- ("acq", "failrelaxed") if is_cxchg =>
- (Acquire, Monotonic),
- ("acqrel", "failrelaxed") if is_cxchg =>
- (AcquireRelease, Monotonic),
- _ => self.sess().fatal("unknown ordering in atomic intrinsic")
+ ("acq", "failrelaxed") if is_cxchg => (Acquire, Monotonic),
+ ("acqrel", "failrelaxed") if is_cxchg => (AcquireRelease, Monotonic),
+ _ => self.sess().fatal("unknown ordering in atomic intrinsic"),
},
_ => self.sess().fatal("Atomic intrinsic not in correct format"),
};
let invalid_monomorphization = |ty| {
- span_invalid_monomorphization_error(tcx.sess, span,
- &format!("invalid monomorphization of `{}` intrinsic: \
- expected basic integer type, found `{}`", name, ty));
+ span_invalid_monomorphization_error(
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic integer type, found `{}`",
+ name, ty
+ ),
+ );
};
match split[1] {
args[2].immediate(),
order,
failorder,
- weak);
+ weak,
+ );
let val = self.extract_value(pair, 0);
let success = self.extract_value(pair, 1);
let success = self.zext(success, self.type_bool());
args[1].immediate(),
args[0].immediate(),
order,
- size
+ size,
);
return;
} else {
// These are all AtomicRMW ops
op => {
let atom_op = match op {
- "xchg" => AtomicRmwBinOp::AtomicXchg,
- "xadd" => AtomicRmwBinOp::AtomicAdd,
- "xsub" => AtomicRmwBinOp::AtomicSub,
- "and" => AtomicRmwBinOp::AtomicAnd,
- "nand" => AtomicRmwBinOp::AtomicNand,
- "or" => AtomicRmwBinOp::AtomicOr,
- "xor" => AtomicRmwBinOp::AtomicXor,
- "max" => AtomicRmwBinOp::AtomicMax,
- "min" => AtomicRmwBinOp::AtomicMin,
- "umax" => AtomicRmwBinOp::AtomicUMax,
- "umin" => AtomicRmwBinOp::AtomicUMin,
- _ => self.sess().fatal("unknown atomic operation")
+ "xchg" => AtomicRmwBinOp::AtomicXchg,
+ "xadd" => AtomicRmwBinOp::AtomicAdd,
+ "xsub" => AtomicRmwBinOp::AtomicSub,
+ "and" => AtomicRmwBinOp::AtomicAnd,
+ "nand" => AtomicRmwBinOp::AtomicNand,
+ "or" => AtomicRmwBinOp::AtomicOr,
+ "xor" => AtomicRmwBinOp::AtomicXor,
+ "max" => AtomicRmwBinOp::AtomicMax,
+ "min" => AtomicRmwBinOp::AtomicMin,
+ "umax" => AtomicRmwBinOp::AtomicUMax,
+ "umin" => AtomicRmwBinOp::AtomicUMin,
+ _ => self.sess().fatal("unknown atomic operation"),
};
let ty = substs.type_at(0);
atom_op,
args[0].immediate(),
args[1].immediate(),
- order
+ order,
)
} else {
return invalid_monomorphization(ty);
self.store(llval, ptr, result.align);
} else {
OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
- .val.store(self, result);
+ .val
+ .store(self, result);
}
}
}
) {
let (size, align) = bx.size_and_align_of(ty);
let size = bx.mul(bx.const_usize(size.bytes()), count);
- let flags = if volatile {
- MemFlags::VOLATILE
- } else {
- MemFlags::empty()
- };
+ let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
if allow_overlap {
bx.memmove(dst, align, src, align, size, flags);
} else {
ty: Ty<'tcx>,
dst: &'ll Value,
val: &'ll Value,
- count: &'ll Value
+ count: &'ll Value,
) {
let (size, align) = bx.size_and_align_of(ty);
let size = bx.mul(bx.const_usize(size.bytes()), count);
- let flags = if volatile {
- MemFlags::VOLATILE
- } else {
- MemFlags::empty()
- };
+ let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
bx.memset(dst, val, size, align, flags);
}
output,
false,
hir::Unsafety::Unsafe,
- Abi::Rust
+ Abi::Rust,
));
let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]);
let llfn = cx.declare_fn(name, &fn_abi);
tcx.mk_unit(),
false,
hir::Unsafety::Unsafe,
- Abi::Rust
+ Abi::Rust,
)));
let output = tcx.types.i32;
let rust_try = gen_fn(cx, "__rust_try", vec![fn_ty, i8p, i8p], output, codegen);
args: &[OperandRef<'tcx, &'ll Value>],
ret_ty: Ty<'tcx>,
llret_ty: &'ll Type,
- span: Span
+ span: Span,
) -> Result<&'ll Value, ()> {
// macros for error handling:
macro_rules! emit_error {
macro_rules! require_simd {
($ty: expr, $position: expr) => {
require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty)
- }
+ };
}
let tcx = bx.tcx();
- let sig = tcx.normalize_erasing_late_bound_regions(
- ty::ParamEnv::reveal_all(),
- &callee_ty.fn_sig(tcx),
- );
+ let sig = tcx
+ .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &callee_ty.fn_sig(tcx));
let arg_tys = sig.inputs();
if name == "simd_select_bitmask" {
};
require_simd!(arg_tys[1], "argument");
let v_len = arg_tys[1].simd_size(tcx);
- require!(m_len == v_len,
- "mismatched lengths: mask length `{}` != other vector length `{}`",
- m_len, v_len
+ require!(
+ m_len == v_len,
+ "mismatched lengths: mask length `{}` != other vector length `{}`",
+ m_len,
+ v_len
);
let i1 = bx.type_i1();
let i1xn = bx.type_vector(i1, m_len);
"simd_le" => Some(hir::BinOpKind::Le),
"simd_gt" => Some(hir::BinOpKind::Gt),
"simd_ge" => Some(hir::BinOpKind::Ge),
- _ => None
+ _ => None,
};
if let Some(cmp_op) = comparison {
require_simd!(ret_ty, "return");
let out_len = ret_ty.simd_size(tcx);
- require!(in_len == out_len,
- "expected return type with length {} (same as input type `{}`), \
+ require!(
+ in_len == out_len,
+ "expected return type with length {} (same as input type `{}`), \
found `{}` with length {}",
- in_len, in_ty,
- ret_ty, out_len);
- require!(bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
- "expected return type with integer elements, found `{}` with non-integer `{}`",
- ret_ty,
- ret_ty.simd_type(tcx));
-
- return Ok(compare_simd_types(bx,
- args[0].immediate(),
- args[1].immediate(),
- in_elem,
- llret_ty,
- cmp_op))
+ in_len,
+ in_ty,
+ ret_ty,
+ out_len
+ );
+ require!(
+ bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
+ "expected return type with integer elements, found `{}` with non-integer `{}`",
+ ret_ty,
+ ret_ty.simd_type(tcx)
+ );
+
+ return Ok(compare_simd_types(
+ bx,
+ args[0].immediate(),
+ args[1].immediate(),
+ in_elem,
+ llret_ty,
+ cmp_op,
+ ));
}
if name.starts_with("simd_shuffle") {
- let n: u64 = name["simd_shuffle".len()..].parse().unwrap_or_else(|_|
- span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?"));
+ let n: u64 = name["simd_shuffle".len()..].parse().unwrap_or_else(|_| {
+ span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
+ });
require_simd!(ret_ty, "return");
let out_len = ret_ty.simd_size(tcx);
- require!(out_len == n,
- "expected return type of length {}, found `{}` with length {}",
- n, ret_ty, out_len);
- require!(in_elem == ret_ty.simd_type(tcx),
- "expected return element type `{}` (element of input `{}`), \
+ require!(
+ out_len == n,
+ "expected return type of length {}, found `{}` with length {}",
+ n,
+ ret_ty,
+ out_len
+ );
+ require!(
+ in_elem == ret_ty.simd_type(tcx),
+ "expected return element type `{}` (element of input `{}`), \
found `{}` with element type `{}`",
- in_elem, in_ty,
- ret_ty, ret_ty.simd_type(tcx));
+ in_elem,
+ in_ty,
+ ret_ty,
+ ret_ty.simd_type(tcx)
+ );
let total_len = u128::from(in_len) * 2;
None
}
Some(idx) if idx >= total_len => {
- emit_error!("shuffle index #{} is out of bounds (limit {})",
- arg_idx, total_len);
+ emit_error!(
+ "shuffle index #{} is out of bounds (limit {})",
+ arg_idx,
+ total_len
+ );
None
}
Some(idx) => Some(bx.const_i32(idx as i32)),
.collect();
let indices = match indices {
Some(i) => i,
- None => return Ok(bx.const_null(llret_ty))
+ None => return Ok(bx.const_null(llret_ty)),
};
- return Ok(bx.shuffle_vector(args[0].immediate(),
- args[1].immediate(),
- bx.const_vector(&indices)))
+ return Ok(bx.shuffle_vector(
+ args[0].immediate(),
+ args[1].immediate(),
+ bx.const_vector(&indices),
+ ));
}
if name == "simd_insert" {
- require!(in_elem == arg_tys[2],
- "expected inserted type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, arg_tys[2]);
- return Ok(bx.insert_element(args[0].immediate(),
- args[2].immediate(),
- args[1].immediate()))
+ require!(
+ in_elem == arg_tys[2],
+ "expected inserted type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ arg_tys[2]
+ );
+ return Ok(bx.insert_element(
+ args[0].immediate(),
+ args[2].immediate(),
+ args[1].immediate(),
+ ));
}
if name == "simd_extract" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return Ok(bx.extract_element(args[0].immediate(), args[1].immediate()))
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
+ return Ok(bx.extract_element(args[0].immediate(), args[1].immediate()));
}
if name == "simd_select" {
let m_len = in_len;
require_simd!(arg_tys[1], "argument");
let v_len = arg_tys[1].simd_size(tcx);
- require!(m_len == v_len,
- "mismatched lengths: mask length `{}` != other vector length `{}`",
- m_len, v_len
+ require!(
+ m_len == v_len,
+ "mismatched lengths: mask length `{}` != other vector length `{}`",
+ m_len,
+ v_len
);
match m_elem_ty.kind {
- ty::Int(_) => {},
- _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty)
+ ty::Int(_) => {}
+ _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty),
}
// truncate the mask to a vector of i1s
let i1 = bx.type_i1();
// trailing bits.
let expected_int_bits = in_len.max(8);
match ret_ty.kind {
- ty::Uint(i) if i.bit_width() == Some(expected_int_bits as usize) => (),
- _ => return_error!(
- "bitmask `{}`, expected `u{}`",
- ret_ty, expected_int_bits
- ),
+ ty::Uint(i) if i.bit_width() == Some(expected_int_bits as usize) => (),
+ _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits),
}
// Integer vector <i{in_bitwidth} x in_len>:
let (i_xn, in_elem_bitwidth) = match in_elem.kind {
ty::Int(i) => (
args[0].immediate(),
- i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _)
+ i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _),
),
ty::Uint(i) => (
args[0].immediate(),
- i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _)
+ i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _),
),
_ => return_error!(
"vector argument `{}`'s element type `{}`, expected integer element type",
- in_ty, in_elem
+ in_ty,
+ in_elem
),
};
// Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
- let shift_indices = vec![
- bx.cx.const_int(bx.type_ix(in_elem_bitwidth as _), (in_elem_bitwidth - 1) as _);
- in_len as _
- ];
+ let shift_indices =
+ vec![
+ bx.cx.const_int(bx.type_ix(in_elem_bitwidth as _), (in_elem_bitwidth - 1) as _);
+ in_len as _
+ ];
let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
// Truncate vector to an <i1 x N>
let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len as _));
return_error!(
"unsupported floating-point vector `{}` with length `{}` \
out-of-range [2, 16]",
- in_ty, in_len);
+ in_ty,
+ in_len
+ );
}
"f32"
- },
+ }
ty::Float(f) if f.bit_width() == 64 => {
if in_len < 2 || in_len > 8 {
- return_error!("unsupported floating-point vector `{}` with length `{}` \
+ return_error!(
+ "unsupported floating-point vector `{}` with length `{}` \
out-of-range [2, 8]",
- in_ty, in_len);
+ in_ty,
+ in_len
+ );
}
"f64"
- },
+ }
ty::Float(f) => {
- return_error!("unsupported element type `{}` of floating-point vector `{}`",
- f.name_str(), in_ty);
- },
+ return_error!(
+ "unsupported element type `{}` of floating-point vector `{}`",
+ f.name_str(),
+ in_ty
+ );
+ }
_ => {
return_error!("`{}` is not a floating-point type", in_ty);
}
let llvm_name = &format!("llvm.{0}.v{1}{2}", name, in_len, ety);
let intrinsic = bx.get_intrinsic(&llvm_name);
- let c = bx.call(intrinsic,
- &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
- None);
+ let c =
+ bx.call(intrinsic, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) };
Ok(c)
}
}
}
- fn llvm_vector_ty(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64,
- mut no_pointers: usize) -> &'ll Type {
+ fn llvm_vector_ty(
+ cx: &CodegenCx<'ll, '_>,
+ elem_ty: Ty<'_>,
+ vec_len: u64,
+ mut no_pointers: usize,
+ ) -> &'ll Type {
// FIXME: use cx.layout_of(ty).llvm_type() ?
let mut elem_ty = match elem_ty.kind {
- ty::Int(v) => cx.type_int_from_ty( v),
- ty::Uint(v) => cx.type_uint_from_ty( v),
- ty::Float(v) => cx.type_float_from_ty( v),
+ ty::Int(v) => cx.type_int_from_ty(v),
+ ty::Uint(v) => cx.type_uint_from_ty(v),
+ ty::Float(v) => cx.type_float_from_ty(v),
_ => unreachable!(),
};
while no_pointers > 0 {
cx.type_vector(elem_ty, vec_len)
}
-
if name == "simd_gather" {
// simd_gather(values: <N x T>, pointers: <N x *_ T>,
// mask: <N x i{M}>) -> <N x T>
require_simd!(ret_ty, "return");
// Of the same length:
- require!(in_len == arg_tys[1].simd_size(tcx),
- "expected {} argument with length {} (same as input type `{}`), \
- found `{}` with length {}", "second", in_len, in_ty, arg_tys[1],
- arg_tys[1].simd_size(tcx));
- require!(in_len == arg_tys[2].simd_size(tcx),
- "expected {} argument with length {} (same as input type `{}`), \
- found `{}` with length {}", "third", in_len, in_ty, arg_tys[2],
- arg_tys[2].simd_size(tcx));
+ require!(
+ in_len == arg_tys[1].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "second",
+ in_len,
+ in_ty,
+ arg_tys[1],
+ arg_tys[1].simd_size(tcx)
+ );
+ require!(
+ in_len == arg_tys[2].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "third",
+ in_len,
+ in_ty,
+ arg_tys[2],
+ arg_tys[2].simd_size(tcx)
+ );
// The return type must match the first argument type
- require!(ret_ty == in_ty,
- "expected return type `{}`, found `{}`",
- in_ty, ret_ty);
+ require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty);
// This counts how many pointers
fn ptr_count(t: Ty<'_>) -> usize {
// The second argument must be a simd vector with an element type that's a pointer
// to the element type of the first argument
let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind {
- ty::RawPtr(p) if p.ty == in_elem => (ptr_count(arg_tys[1].simd_type(tcx)),
- non_ptr(arg_tys[1].simd_type(tcx))),
+ ty::RawPtr(p) if p.ty == in_elem => {
+ (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx)))
+ }
_ => {
- require!(false, "expected element type `{}` of second argument `{}` \
+ require!(
+ false,
+ "expected element type `{}` of second argument `{}` \
to be a pointer to the element type `{}` of the first \
argument `{}`, found `{}` != `*_ {}`",
- arg_tys[1].simd_type(tcx), arg_tys[1], in_elem, in_ty,
- arg_tys[1].simd_type(tcx), in_elem);
+ arg_tys[1].simd_type(tcx),
+ arg_tys[1],
+ in_elem,
+ in_ty,
+ arg_tys[1].simd_type(tcx),
+ in_elem
+ );
unreachable!();
}
};
match arg_tys[2].simd_type(tcx).kind {
ty::Int(_) => (),
_ => {
- require!(false, "expected element type `{}` of third argument `{}` \
+ require!(
+ false,
+ "expected element type `{}` of third argument `{}` \
to be a signed integer type",
- arg_tys[2].simd_type(tcx), arg_tys[2]);
+ arg_tys[2].simd_type(tcx),
+ arg_tys[2]
+ );
}
}
let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
- let llvm_intrinsic = format!("llvm.masked.gather.{}.{}",
- llvm_elem_vec_str, llvm_pointer_vec_str);
- let f = bx.declare_cfn(&llvm_intrinsic,
- bx.type_func(&[
- llvm_pointer_vec_ty,
- alignment_ty,
- mask_ty,
- llvm_elem_vec_ty], llvm_elem_vec_ty));
+ let llvm_intrinsic =
+ format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
+ let f = bx.declare_cfn(
+ &llvm_intrinsic,
+ bx.type_func(
+ &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty],
+ llvm_elem_vec_ty,
+ ),
+ );
llvm::SetUnnamedAddr(f, false);
- let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()],
- None);
+ let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None);
return Ok(v);
}
require_simd!(arg_tys[2], "third");
// Of the same length:
- require!(in_len == arg_tys[1].simd_size(tcx),
- "expected {} argument with length {} (same as input type `{}`), \
- found `{}` with length {}", "second", in_len, in_ty, arg_tys[1],
- arg_tys[1].simd_size(tcx));
- require!(in_len == arg_tys[2].simd_size(tcx),
- "expected {} argument with length {} (same as input type `{}`), \
- found `{}` with length {}", "third", in_len, in_ty, arg_tys[2],
- arg_tys[2].simd_size(tcx));
+ require!(
+ in_len == arg_tys[1].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "second",
+ in_len,
+ in_ty,
+ arg_tys[1],
+ arg_tys[1].simd_size(tcx)
+ );
+ require!(
+ in_len == arg_tys[2].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "third",
+ in_len,
+ in_ty,
+ arg_tys[2],
+ arg_tys[2].simd_size(tcx)
+ );
// This counts how many pointers
fn ptr_count(t: Ty<'_>) -> usize {
// The second argument must be a simd vector with an element type that's a pointer
// to the element type of the first argument
let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind {
- ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut
- => (ptr_count(arg_tys[1].simd_type(tcx)),
- non_ptr(arg_tys[1].simd_type(tcx))),
+ ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => {
+ (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx)))
+ }
_ => {
- require!(false, "expected element type `{}` of second argument `{}` \
+ require!(
+ false,
+ "expected element type `{}` of second argument `{}` \
to be a pointer to the element type `{}` of the first \
argument `{}`, found `{}` != `*mut {}`",
- arg_tys[1].simd_type(tcx), arg_tys[1], in_elem, in_ty,
- arg_tys[1].simd_type(tcx), in_elem);
+ arg_tys[1].simd_type(tcx),
+ arg_tys[1],
+ in_elem,
+ in_ty,
+ arg_tys[1].simd_type(tcx),
+ in_elem
+ );
unreachable!();
}
};
match arg_tys[2].simd_type(tcx).kind {
ty::Int(_) => (),
_ => {
- require!(false, "expected element type `{}` of third argument `{}` \
+ require!(
+ false,
+ "expected element type `{}` of third argument `{}` \
to be a signed integer type",
- arg_tys[2].simd_type(tcx), arg_tys[2]);
+ arg_tys[2].simd_type(tcx),
+ arg_tys[2]
+ );
}
}
let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
- let llvm_intrinsic = format!("llvm.masked.scatter.{}.{}",
- llvm_elem_vec_str, llvm_pointer_vec_str);
- let f = bx.declare_cfn(&llvm_intrinsic,
- bx.type_func(&[llvm_elem_vec_ty,
- llvm_pointer_vec_ty,
- alignment_ty,
- mask_ty], ret_t));
+ let llvm_intrinsic =
+ format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
+ let f = bx.declare_cfn(
+ &llvm_intrinsic,
+ bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t),
+ );
llvm::SetUnnamedAddr(f, false);
- let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask],
- None);
+ let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None);
return Ok(v);
}
macro_rules! arith_red {
($name:tt : $integer_reduce:ident, $float_reduce:ident, $ordered:expr) => {
if name == $name {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
return match in_elem.kind {
ty::Int(_) | ty::Uint(_) => {
let r = bx.$integer_reduce(args[0].immediate());
} else {
Ok(bx.$integer_reduce(args[0].immediate()))
}
- },
+ }
ty::Float(f) => {
let acc = if $ordered {
// ordered arithmetic reductions take an accumulator
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), identity_acc),
64 => bx.const_real(bx.type_f64(), identity_acc),
- v => {
- return_error!(r#"
+ v => return_error!(
+ r#"
unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
- $name, in_ty, in_elem, v, ret_ty
- )
- }
+ $name,
+ in_ty,
+ in_elem,
+ v,
+ ret_ty
+ ),
}
};
Ok(bx.$float_reduce(acc, args[0].immediate()))
}
- _ => {
- return_error!(
- "unsupported {} from `{}` with element `{}` to `{}`",
- $name, in_ty, in_elem, ret_ty
- )
- },
- }
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ $name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
+ };
}
- }
+ };
}
arith_red!("simd_reduce_add_ordered": vector_reduce_add, vector_reduce_fadd, true);
macro_rules! minmax_red {
($name:tt: $int_red:ident, $float_red:ident) => {
if name == $name {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
return match in_elem.kind {
- ty::Int(_i) => {
- Ok(bx.$int_red(args[0].immediate(), true))
- },
- ty::Uint(_u) => {
- Ok(bx.$int_red(args[0].immediate(), false))
- },
- ty::Float(_f) => {
- Ok(bx.$float_red(args[0].immediate()))
- }
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- $name, in_ty, in_elem, ret_ty)
- },
- }
+ ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
+ ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
+ ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ $name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
+ };
}
-
- }
+ };
}
minmax_red!("simd_reduce_min": vector_reduce_min, vector_reduce_fmin);
($name:tt : $red:ident, $boolean:expr) => {
if name == $name {
let input = if !$boolean {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
args[0].immediate()
} else {
match in_elem.kind {
- ty::Int(_) | ty::Uint(_) => {},
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- $name, in_ty, in_elem, ret_ty)
- }
+ ty::Int(_) | ty::Uint(_) => {}
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ $name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
}
// boolean reductions operate on vectors of i1s:
return match in_elem.kind {
ty::Int(_) | ty::Uint(_) => {
let r = bx.$red(input);
- Ok(
- if !$boolean {
- r
- } else {
- bx.zext(r, bx.type_bool())
- }
- )
- },
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- $name, in_ty, in_elem, ret_ty)
- },
- }
+ Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
+ }
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ $name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
+ };
}
- }
+ };
}
bitwise_red!("simd_reduce_and": vector_reduce_and, false);
if name == "simd_cast" {
require_simd!(ret_ty, "return");
let out_len = ret_ty.simd_size(tcx);
- require!(in_len == out_len,
- "expected return type with length {} (same as input type `{}`), \
+ require!(
+ in_len == out_len,
+ "expected return type with length {} (same as input type `{}`), \
found `{}` with length {}",
- in_len, in_ty,
- ret_ty, out_len);
+ in_len,
+ in_ty,
+ ret_ty,
+ out_len
+ );
// casting cares about nominal type, not just structural type
let out_elem = ret_ty.simd_type(tcx);
- if in_elem == out_elem { return Ok(args[0].immediate()); }
+ if in_elem == out_elem {
+ return Ok(args[0].immediate());
+ }
- enum Style { Float, Int(/* is signed? */ bool), Unsupported }
+ enum Style {
+ Float,
+ Int(/* is signed? */ bool),
+ Unsupported,
+ }
let (in_style, in_width) = match in_elem.kind {
// vectors of pointer-sized integers should've been
ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()),
ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()),
ty::Float(f) => (Style::Float, f.bit_width()),
- _ => (Style::Unsupported, 0)
+ _ => (Style::Unsupported, 0),
};
let (out_style, out_width) = match out_elem.kind {
ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()),
ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()),
ty::Float(f) => (Style::Float, f.bit_width()),
- _ => (Style::Unsupported, 0)
+ _ => (Style::Unsupported, 0),
};
match (in_style, out_style) {
return Ok(match in_width.cmp(&out_width) {
Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
Ordering::Equal => args[0].immediate(),
- Ordering::Less => if in_is_signed {
- bx.sext(args[0].immediate(), llret_ty)
- } else {
- bx.zext(args[0].immediate(), llret_ty)
+ Ordering::Less => {
+ if in_is_signed {
+ bx.sext(args[0].immediate(), llret_ty)
+ } else {
+ bx.zext(args[0].immediate(), llret_ty)
+ }
}
- })
+ });
}
(Style::Int(in_is_signed), Style::Float) => {
return Ok(if in_is_signed {
bx.sitofp(args[0].immediate(), llret_ty)
} else {
bx.uitofp(args[0].immediate(), llret_ty)
- })
+ });
}
(Style::Float, Style::Int(out_is_signed)) => {
return Ok(if out_is_signed {
bx.fptosi(args[0].immediate(), llret_ty)
} else {
bx.fptoui(args[0].immediate(), llret_ty)
- })
+ });
}
(Style::Float, Style::Float) => {
return Ok(match in_width.cmp(&out_width) {
Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
Ordering::Equal => args[0].immediate(),
- Ordering::Less => bx.fpext(args[0].immediate(), llret_ty)
- })
+ Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
+ });
}
- _ => {/* Unsupported. Fallthrough. */}
+ _ => { /* Unsupported. Fallthrough. */ }
}
- require!(false,
- "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
- in_ty, in_elem,
- ret_ty, out_elem);
+ require!(
+ false,
+ "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
+ in_ty,
+ in_elem,
+ ret_ty,
+ out_elem
+ );
}
macro_rules! arith {
($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
let is_add = name == "simd_saturating_add";
let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
let (signed, elem_width, elem_ty) = match in_elem.kind {
- ty::Int(i) =>
- (
- true,
- i.bit_width().unwrap_or(ptr_bits),
- bx.cx.type_int_from_ty(i)
- ),
- ty::Uint(i) =>
- (
- false,
- i.bit_width().unwrap_or(ptr_bits),
- bx.cx.type_uint_from_ty(i)
- ),
+ ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
+ ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
_ => {
return_error!(
"expected element type `{}` of vector type `{}` \
to be a signed or unsigned integer type",
- arg_tys[0].simd_type(tcx), arg_tys[0]
+ arg_tys[0].simd_type(tcx),
+ arg_tys[0]
);
}
};
"llvm.{}{}.sat.v{}i{}",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" },
- in_len, elem_width
+ in_len,
+ elem_width
);
let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
- let f = bx.declare_cfn(
- &llvm_intrinsic,
- bx.type_func(&[vec_ty, vec_ty], vec_ty)
- );
+ let f = bx.declare_cfn(&llvm_intrinsic, bx.type_func(&[vec_ty, vec_ty], vec_ty));
llvm::SetUnnamedAddr(f, false);
let v = bx.call(f, &[lhs, rhs], None);
return Ok(v);
// stuffs.
fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> {
match ty.kind {
- ty::Int(t) => Some((match t {
- ast::IntTy::Isize => cx.tcx.sess.target.ptr_width as u64,
- ast::IntTy::I8 => 8,
- ast::IntTy::I16 => 16,
- ast::IntTy::I32 => 32,
- ast::IntTy::I64 => 64,
- ast::IntTy::I128 => 128,
- }, true)),
- ty::Uint(t) => Some((match t {
- ast::UintTy::Usize => cx.tcx.sess.target.ptr_width as u64,
- ast::UintTy::U8 => 8,
- ast::UintTy::U16 => 16,
- ast::UintTy::U32 => 32,
- ast::UintTy::U64 => 64,
- ast::UintTy::U128 => 128,
- }, false)),
+ ty::Int(t) => Some((
+ match t {
+ ast::IntTy::Isize => cx.tcx.sess.target.ptr_width as u64,
+ ast::IntTy::I8 => 8,
+ ast::IntTy::I16 => 16,
+ ast::IntTy::I32 => 32,
+ ast::IntTy::I64 => 64,
+ ast::IntTy::I128 => 128,
+ },
+ true,
+ )),
+ ty::Uint(t) => Some((
+ match t {
+ ast::UintTy::Usize => cx.tcx.sess.target.ptr_width as u64,
+ ast::UintTy::U8 => 8,
+ ast::UintTy::U16 => 16,
+ ast::UintTy::U32 => 32,
+ ast::UintTy::U64 => 64,
+ ast::UintTy::U128 => 128,
+ },
+ false,
+ )),
_ => None,
}
}
projects / rust.git / blobdiff