return Ok(bx.extract_element(args[0].immediate(), args[1].immediate()))
}
- if name == "simd_reduce_add" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return match in_elem.sty {
- ty::TyInt(_i) => {
- Ok(bx.vector_reduce_add(args[0].immediate()))
- },
- ty::TyUint(_u) => {
- Ok(bx.vector_reduce_add(args[0].immediate()))
- },
- ty::TyFloat(f) => {
- // undef as accumulator makes the reduction unordered:
- let acc = match f.bit_width() {
- 32 => C_undef(Type::f32(bx.cx)),
- 64 => C_undef(Type::f64(bx.cx)),
- v => {
- return_error!(
- "unsupported {} from `{}` with element `{}` of size `{}` to `{}`",
- "simd_reduce_add", in_ty, in_elem, v, ret_ty)
+ macro_rules! arith_red {
+ ($name:tt : $integer_reduce:ident, $float_reduce:ident, $ordered:expr) => {
+ if name == $name {
+ require!(ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem, in_ty, ret_ty);
+ return match in_elem.sty {
+ ty::TyInt(_) | ty::TyUint(_) => {
+ let r = bx.$integer_reduce(args[0].immediate());
+ if $ordered {
+ // if overflow occurs, the result is the
+ // mathematical result modulo 2^n:
+ if name.contains("mul") {
+ Ok(bx.mul(args[1].immediate(), r))
+ } else {
+ Ok(bx.add(args[1].immediate(), r))
+ }
+ } else {
+ Ok(bx.$integer_reduce(args[0].immediate()))
+ }
+ },
+ ty::TyFloat(f) => {
+ // ordered arithmetic reductions take an accumulator
+ let acc = if $ordered {
+ args[1].immediate()
+ } else {
+ // unordered arithmetic reductions do not:
+ match f.bit_width() {
+ 32 => C_undef(Type::f32(bx.cx)),
+ 64 => C_undef(Type::f64(bx.cx)),
+ v => {
+ return_error!(
+ "unsupported {} from `{}` with element `{}` of size `{}` to `{}`",
+ $name, in_ty, in_elem, v, ret_ty
+ )
+ }
+ }
+
+ };
+ Ok(bx.$float_reduce(acc, args[0].immediate()))
}
- };
- Ok(bx.vector_reduce_fadd_fast(acc, args[0].immediate()))
+ _ => {
+ return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ $name, in_ty, in_elem, ret_ty
+ )
+ },
+ }
}
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_add", in_ty, in_elem, ret_ty)
- },
}
}
- if name == "simd_reduce_mul" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return match in_elem.sty {
- ty::TyInt(_i) => {
- Ok(bx.vector_reduce_mul(args[0].immediate()))
- },
- ty::TyUint(_u) => {
- Ok(bx.vector_reduce_mul(args[0].immediate()))
- },
- ty::TyFloat(f) => {
- // undef as accumulator makes the reduction unordered:
- let acc = match f.bit_width() {
- 32 => C_undef(Type::f32(bx.cx)),
- 64 => C_undef(Type::f64(bx.cx)),
- v => {
- return_error!(
- "unsupported {} from `{}` with element `{}` of size `{}` to `{}`",
- "simd_reduce_mul", in_ty, in_elem, v, ret_ty)
+ arith_red!("simd_reduce_add_ordered": vector_reduce_add, vector_reduce_fadd_fast, true);
+ arith_red!("simd_reduce_mul_ordered": vector_reduce_mul, vector_reduce_fmul_fast, true);
+ arith_red!("simd_reduce_add_unordered": vector_reduce_add, vector_reduce_fadd_fast, false);
+ arith_red!("simd_reduce_mul_unordered": vector_reduce_mul, vector_reduce_fmul_fast, false);
+
+ macro_rules! minmax_red {
+ ($name:tt: $int_red:ident, $float_red:ident) => {
+ if name == $name {
+ require!(ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem, in_ty, ret_ty);
+ return match in_elem.sty {
+ ty::TyInt(_i) => {
+ Ok(bx.$int_red(args[0].immediate(), true))
+ },
+ ty::TyUint(_u) => {
+ Ok(bx.$int_red(args[0].immediate(), false))
+ },
+ ty::TyFloat(_f) => {
+ Ok(bx.$float_red(args[0].immediate()))
}
- };
- Ok(bx.vector_reduce_fmul_fast(acc, args[0].immediate()))
+ _ => {
+ return_error!("unsupported {} from `{}` with element `{}` to `{}`",
+ $name, in_ty, in_elem, ret_ty)
+ },
+ }
}
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_mul", in_ty, in_elem, ret_ty)
- },
- }
- }
- if name == "simd_reduce_min" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return match in_elem.sty {
- ty::TyInt(_i) => {
- Ok(bx.vector_reduce_min(args[0].immediate(), true))
- },
- ty::TyUint(_u) => {
- Ok(bx.vector_reduce_min(args[0].immediate(), false))
- },
- ty::TyFloat(_f) => {
- Ok(bx.vector_reduce_fmin_fast(args[0].immediate()))
- }
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_min", in_ty, in_elem, ret_ty)
- },
}
}
- if name == "simd_reduce_max" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return match in_elem.sty {
- ty::TyInt(_i) => {
- Ok(bx.vector_reduce_max(args[0].immediate(), true))
- },
- ty::TyUint(_u) => {
- Ok(bx.vector_reduce_max(args[0].immediate(), false))
- },
- ty::TyFloat(_f) => {
- Ok(bx.vector_reduce_fmax_fast(args[0].immediate()))
- }
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_max", in_ty, in_elem, ret_ty)
- },
- }
- }
+ minmax_red!("simd_reduce_min": vector_reduce_min, vector_reduce_fmin);
+ minmax_red!("simd_reduce_max": vector_reduce_max, vector_reduce_fmax);
- if name == "simd_reduce_and" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return match in_elem.sty {
- ty::TyInt(_i) => {
- Ok(bx.vector_reduce_and(args[0].immediate()))
- },
- ty::TyUint(_u) => {
- Ok(bx.vector_reduce_and(args[0].immediate()))
- },
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_and", in_ty, in_elem, ret_ty)
- },
- }
- }
+ minmax_red!("simd_reduce_min_nanless": vector_reduce_min, vector_reduce_fmin_fast);
+ minmax_red!("simd_reduce_max_nanless": vector_reduce_max, vector_reduce_fmax_fast);
- if name == "simd_reduce_or" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return match in_elem.sty {
- ty::TyInt(_i) => {
- Ok(bx.vector_reduce_or(args[0].immediate()))
- },
- ty::TyUint(_u) => {
- Ok(bx.vector_reduce_or(args[0].immediate()))
- },
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_or", in_ty, in_elem, ret_ty)
- },
- }
- }
-
- if name == "simd_reduce_xor" {
- require!(ret_ty == in_elem,
- "expected return type `{}` (element of input `{}`), found `{}`",
- in_elem, in_ty, ret_ty);
- return match in_elem.sty {
- ty::TyInt(_i) => {
- Ok(bx.vector_reduce_xor(args[0].immediate()))
- },
- ty::TyUint(_u) => {
- Ok(bx.vector_reduce_xor(args[0].immediate()))
- },
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_xor", in_ty, in_elem, ret_ty)
- },
+ macro_rules! bitwise_red {
+ ($name:tt : $red:ident, $boolean:expr) => {
+ if name == $name {
+ let input = if !$boolean {
+ require!(ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem, in_ty, ret_ty);
+ args[0].immediate()
+ } else {
+ // boolean reductions operate on vectors of i1s:
+ let i1 = Type::i1(bx.cx);
+ let i1xn = Type::vector(&i1, in_len as u64);
+ bx.trunc(args[0].immediate(), i1xn)
+ };
+ return match in_elem.sty {
+ ty::TyInt(_) | ty::TyUint(_) => {
+ let r = bx.$red(input);
+ Ok(
+ if !$boolean {
+ r
+ } else {
+ bx.zext(r, Type::bool(bx.cx))
+ }
+ )
+ },
+ _ => {
+ return_error!("unsupported {} from `{}` with element `{}` to `{}`",
+ $name, in_ty, in_elem, ret_ty)
+ },
+ }
+ }
}
}
- if name == "simd_reduce_all" {
- //require!(ret_ty == in_elem,
- // "expected return type `{}` (element of input `{}`), found `{}`",
- // in_elem, in_ty, ret_ty);
- let i1 = Type::i1(bx.cx);
- let i1xn = Type::vector(&i1, in_len as u64);
- let v = bx.trunc(args[0].immediate(), i1xn);
-
- let red = match in_elem.sty {
- ty::TyInt(_i) => {
- bx.vector_reduce_and(v)
- },
- ty::TyUint(_u) => {
- bx.vector_reduce_and(v)
- },
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_and", in_ty, in_elem, ret_ty)
- },
- };
- return Ok(bx.zext(red, Type::bool(bx.cx)));
- }
-
- if name == "simd_reduce_any" {
- //require!(ret_ty == in_elem,
- // "expected return type `{}` (element of input `{}`), found `{}`",
- // in_elem, in_ty, ret_ty);
- let i1 = Type::i1(bx.cx);
- let i1xn = Type::vector(&i1, in_len as u64);
- let v = bx.trunc(args[0].immediate(), i1xn);
-
- let red = match in_elem.sty {
- ty::TyInt(_i) => {
- bx.vector_reduce_or(v)
- },
- ty::TyUint(_u) => {
- bx.vector_reduce_or(v)
- },
- _ => {
- return_error!("unsupported {} from `{}` with element `{}` to `{}`",
- "simd_reduce_and", in_ty, in_elem, ret_ty)
- },
- };
- return Ok(bx.zext(red, Type::bool(bx.cx)));
- }
-
+ bitwise_red!("simd_reduce_and": vector_reduce_and, false);
+ bitwise_red!("simd_reduce_or": vector_reduce_or, false);
+ bitwise_red!("simd_reduce_xor": vector_reduce_xor, false);
+ bitwise_red!("simd_reduce_all": vector_reduce_and, true);
+ bitwise_red!("simd_reduce_any": vector_reduce_or, true);
if name == "simd_cast" {
require_simd!(ret_ty, "return");
);
extern "platform-intrinsic" {
- fn simd_reduce_add<T, U>(x: T) -> U;
- fn simd_reduce_mul<T, U>(x: T) -> U;
+ fn simd_reduce_add_unordered<T, U>(x: T) -> U;
+ fn simd_reduce_mul_unordered<T, U>(x: T) -> U;
+ fn simd_reduce_add_ordered<T, U>(x: T, acc: U) -> U;
+ fn simd_reduce_mul_ordered<T, U>(x: T, acc: U) -> U;
fn simd_reduce_min<T, U>(x: T) -> U;
fn simd_reduce_max<T, U>(x: T) -> U;
+ fn simd_reduce_min_nanless<T, U>(x: T) -> U;
+ fn simd_reduce_max_nanless<T, U>(x: T) -> U;
fn simd_reduce_and<T, U>(x: T) -> U;
fn simd_reduce_or<T, U>(x: T) -> U;
fn simd_reduce_xor<T, U>(x: T) -> U;
fn main() {
unsafe {
let x = i32x4(1, -2, 3, 4);
- let r: i32 = simd_reduce_add(x);
- assert!(r == 6_i32);
- let r: i32 = simd_reduce_mul(x);
- assert!(r == -24_i32);
+ let r: i32 = simd_reduce_add_unordered(x);
+ assert_eq!(r, 6_i32);
+ let r: i32 = simd_reduce_mul_unordered(x);
+ assert_eq!(r, -24_i32);
+ let r: i32 = simd_reduce_add_ordered(x, -1);
+ assert_eq!(r, 5_i32);
+ let r: i32 = simd_reduce_mul_ordered(x, -1);
+ assert_eq!(r, 24_i32);
+
let r: i32 = simd_reduce_min(x);
- assert!(r == -21_i32);
+ assert_eq!(r, -2_i32);
let r: i32 = simd_reduce_max(x);
- assert!(r == 4_i32);
+ assert_eq!(r, 4_i32);
let x = i32x4(-1, -1, -1, -1);
let r: i32 = simd_reduce_and(x);
- assert!(r == -1_i32);
+ assert_eq!(r, -1_i32);
let r: i32 = simd_reduce_or(x);
- assert!(r == -1_i32);
+ assert_eq!(r, -1_i32);
let r: i32 = simd_reduce_xor(x);
- assert!(r == 0_i32);
+ assert_eq!(r, 0_i32);
let x = i32x4(-1, -1, 0, -1);
let r: i32 = simd_reduce_and(x);
- assert!(r == 0_i32);
+ assert_eq!(r, 0_i32);
let r: i32 = simd_reduce_or(x);
- assert!(r == -1_i32);
+ assert_eq!(r, -1_i32);
let r: i32 = simd_reduce_xor(x);
- assert!(r == -1_i32);
+ assert_eq!(r, -1_i32);
}
unsafe {
let x = u32x4(1, 2, 3, 4);
- let r: u32 = simd_reduce_add(x);
- assert!(r == 10_u32);
- let r: u32 = simd_reduce_mul(x);
- assert!(r == 24_u32);
+ let r: u32 = simd_reduce_add_unordered(x);
+ assert_eq!(r, 10_u32);
+ let r: u32 = simd_reduce_mul_unordered(x);
+ assert_eq!(r, 24_u32);
+ let r: u32 = simd_reduce_add_ordered(x, 1);
+ assert_eq!(r, 11_u32);
+ let r: u32 = simd_reduce_mul_ordered(x, 2);
+ assert_eq!(r, 48_u32);
+
let r: u32 = simd_reduce_min(x);
- assert!(r == 1_u32);
+ assert_eq!(r, 1_u32);
let r: u32 = simd_reduce_max(x);
- assert!(r == 4_u32);
+ assert_eq!(r, 4_u32);
let t = u32::max_value();
let x = u32x4(t, t, t, t);
let r: u32 = simd_reduce_and(x);
- assert!(r == t);
+ assert_eq!(r, t);
let r: u32 = simd_reduce_or(x);
- assert!(r == t);
+ assert_eq!(r, t);
let r: u32 = simd_reduce_xor(x);
- assert!(r == 0_u32);
+ assert_eq!(r, 0_u32);
let x = u32x4(t, t, 0, t);
let r: u32 = simd_reduce_and(x);
- assert!(r == 0_u32);
+ assert_eq!(r, 0_u32);
let r: u32 = simd_reduce_or(x);
- assert!(r == t);
+ assert_eq!(r, t);
let r: u32 = simd_reduce_xor(x);
- assert!(r == t);
+ assert_eq!(r, t);
}
unsafe {
let x = f32x4(1., -2., 3., 4.);
- let r: f32 = simd_reduce_add(x);
- assert!(r == 6_f32);
- let r: f32 = simd_reduce_mul(x);
- assert!(r == -24_f32);
+ let r: f32 = simd_reduce_add_unordered(x);
+ assert_eq!(r, 6_f32);
+ let r: f32 = simd_reduce_mul_unordered(x);
+ assert_eq!(r, -24_f32);
+ // FIXME: only works correctly for accumulator, 0:
+ // https://bugs.llvm.org/show_bug.cgi?id=36734
+ let r: f32 = simd_reduce_add_ordered(x, 0.);
+ assert_eq!(r, 6_f32);
+ // FIXME: only works correctly for accumulator, 1:
+ // https://bugs.llvm.org/show_bug.cgi?id=36734
+ let r: f32 = simd_reduce_mul_ordered(x, 1.);
+ assert_eq!(r, -24_f32);
+
let r: f32 = simd_reduce_min(x);
- assert!(r == -2_f32);
+ assert_eq!(r, -2_f32);
let r: f32 = simd_reduce_max(x);
- assert!(r == 4_f32);
+ assert_eq!(r, 4_f32);
+ let r: f32 = simd_reduce_min_nanless(x);
+ assert_eq!(r, -2_f32);
+ let r: f32 = simd_reduce_max_nanless(x);
+ assert_eq!(r, 4_f32);
}
unsafe {
let x = b8x4(!0, !0, !0, !0);
let r: bool = simd_reduce_all(x);
- //let r: bool = foobar(x);
- assert!(r);
+ assert_eq!(r, true);
let r: bool = simd_reduce_any(x);
- assert!(r);
+ assert_eq!(r, true);
let x = b8x4(!0, !0, 0, !0);
let r: bool = simd_reduce_all(x);
- assert!(!r);
+ assert_eq!(r, false);
let r: bool = simd_reduce_any(x);
- assert!(r);
+ assert_eq!(r, true);
let x = b8x4(0, 0, 0, 0);
let r: bool = simd_reduce_all(x);
- assert!(!r);
+ assert_eq!(r, false);
let r: bool = simd_reduce_any(x);
- assert!(!r);
+ assert_eq!(r, false);
}
}
projects / rust.git / commitdiff