use rustc_middle::ty::{self, Ty};
use rustc_middle::{bug, span_bug};
use rustc_span::{sym, symbol::kw, Span, Symbol};
-use rustc_target::abi::{self, HasDataLayout, Primitive};
+use rustc_target::abi::{self, Align, HasDataLayout, Primitive};
use rustc_target::spec::{HasTargetSpec, PanicStrategy};
use std::cmp::Ordering;
let arg_tys = sig.inputs();
if name == sym::simd_select_bitmask {
- let in_ty = arg_tys[0];
- let m_len = match in_ty.kind() {
- // Note that this `.unwrap()` crashes for isize/usize, that's sort
- // of intentional as there's not currently a use case for that.
- ty::Int(i) => i.bit_width().unwrap(),
- ty::Uint(i) => i.bit_width().unwrap(),
- _ => return_error!("`{}` is not an integral type", in_ty),
- };
require_simd!(arg_tys[1], "argument");
- let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
- require!(
- // Allow masks for vectors with fewer than 8 elements to be
- // represented with a u8 or i8.
- m_len == v_len || (m_len == 8 && v_len < 8),
- "mismatched lengths: mask length `{}` != other vector length `{}`",
- m_len,
- v_len
- );
+ let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+
+ let expected_int_bits = (len.max(8) - 1).next_power_of_two();
+ let expected_bytes = len / 8 + ((len % 8 > 0) as u64);
+
+ let mask_ty = arg_tys[0];
+ let mask = match mask_ty.kind() {
+ ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
+ ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
+ ty::Array(elem, len)
+ if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
+ && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all())
+ == Some(expected_bytes) =>
+ {
+ let place = PlaceRef::alloca(bx, args[0].layout);
+ args[0].val.store(bx, place);
+ let int_ty = bx.type_ix(expected_bytes * 8);
+ let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty));
+ bx.load(int_ty, ptr, Align::ONE)
+ }
+ _ => return_error!(
+ "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`",
+ mask_ty,
+ expected_int_bits,
+ expected_bytes
+ ),
+ };
+
let i1 = bx.type_i1();
- let im = bx.type_ix(v_len);
- let i1xn = bx.type_vector(i1, v_len);
- let m_im = bx.trunc(args[0].immediate(), im);
+ let im = bx.type_ix(len);
+ let i1xn = bx.type_vector(i1, len);
+ let m_im = bx.trunc(mask, im);
let m_i1s = bx.bitcast(m_im, i1xn);
return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
}
if name == sym::simd_bitmask {
// The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
- // vector mask and returns an unsigned integer containing the most
- // significant bit (MSB) of each lane.
-
- // If the vector has less than 8 lanes, a u8 is returned with zeroed
- // trailing bits.
+ // vector mask and returns the most significant bit (MSB) of each lane in the form
+ // of either:
+ // * an unsigned integer
+ // * an array of `u8`
+ // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
+ //
+ // The bit order of the result depends on the byte endianness, LSB-first for little
+ // endian and MSB-first for big endian.
let expected_int_bits = in_len.max(8);
- match ret_ty.kind() {
- ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => (),
- _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits),
- }
+ let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64);
// Integer vector <i{in_bitwidth} x in_len>:
let (i_xn, in_elem_bitwidth) = match in_elem.kind() {
let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len));
// Bitcast <i1 x N> to iN:
let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
- // Zero-extend iN to the bitmask type:
- return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
+
+ match ret_ty.kind() {
+ ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
+ // Zero-extend iN to the bitmask type:
+ return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
+ }
+ ty::Array(elem, len)
+ if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
+ && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all())
+ == Some(expected_bytes) =>
+ {
+ // Zero-extend iN to the array lengh:
+ let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
+
+ // Convert the integer to a byte array
+ let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE);
+ bx.store(ze, ptr, Align::ONE);
+ let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
+ let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty));
+ return Ok(bx.load(array_ty, ptr, Align::ONE));
+ }
+ _ => return_error!(
+ "cannot return `{}`, expected `u{}` or `[u8; {}]`",
+ ret_ty,
+ expected_int_bits,
+ expected_bytes
+ ),
+ }
}
fn simd_simple_float_intrinsic(
let _: u64 = simd_bitmask(m64);
let _: u16 = simd_bitmask(m2);
- //~^ ERROR bitmask `u16`, expected `u8`
+ //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic
let _: u16 = simd_bitmask(m8);
- //~^ ERROR bitmask `u16`, expected `u8`
+ //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic
let _: u32 = simd_bitmask(m16);
- //~^ ERROR bitmask `u32`, expected `u16`
+ //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic
let _: u64 = simd_bitmask(m32);
- //~^ ERROR bitmask `u64`, expected `u32`
+ //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic
let _: u128 = simd_bitmask(m64);
- //~^ ERROR bitmask `u128`, expected `u64`
+ //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic
}
}
-error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u16`, expected `u8`
+error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u16`, expected `u8` or `[u8; 1]`
--> $DIR/generic-bitmask.rs:53:22
|
LL | let _: u16 = simd_bitmask(m2);
| ^^^^^^^^^^^^^^^^
-error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u16`, expected `u8`
+error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u16`, expected `u8` or `[u8; 1]`
--> $DIR/generic-bitmask.rs:56:22
|
LL | let _: u16 = simd_bitmask(m8);
| ^^^^^^^^^^^^^^^^
-error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u32`, expected `u16`
+error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u32`, expected `u16` or `[u8; 2]`
--> $DIR/generic-bitmask.rs:59:22
|
LL | let _: u32 = simd_bitmask(m16);
| ^^^^^^^^^^^^^^^^^
-error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u64`, expected `u32`
+error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u64`, expected `u32` or `[u8; 4]`
--> $DIR/generic-bitmask.rs:62:22
|
LL | let _: u64 = simd_bitmask(m32);
| ^^^^^^^^^^^^^^^^^
-error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u128`, expected `u64`
+error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u128`, expected `u64` or `[u8; 8]`
--> $DIR/generic-bitmask.rs:65:23
|
LL | let _: u128 = simd_bitmask(m64);
#[repr(simd)]
#[derive(Copy, Clone, PartialEq)]
-struct b8x8(pub i8, pub i8, pub i8, pub i8,
- pub i8, pub i8, pub i8, pub i8);
+struct b8x8(pub i8, pub i8, pub i8, pub i8, pub i8, pub i8, pub i8, pub i8);
extern "platform-intrinsic" {
fn simd_select<T, U>(x: T, a: U, b: U) -> U;
//~^ ERROR found non-SIMD `u32`
simd_select_bitmask(0u16, x, x);
- //~^ ERROR mask length `16` != other vector length `4`
- //
+ //~^ ERROR invalid bitmask `u16`, expected `u8` or `[u8; 1]`
+
simd_select_bitmask(0u8, 1u32, 2u32);
//~^ ERROR found non-SIMD `u32`
simd_select_bitmask(0.0f32, x, x);
- //~^ ERROR `f32` is not an integral type
+ //~^ ERROR invalid bitmask `f32`, expected `u8` or `[u8; 1]`
simd_select_bitmask("x", x, x);
- //~^ ERROR `&str` is not an integral type
+ //~^ ERROR invalid bitmask `&str`, expected `u8` or `[u8; 1]`
}
}
error[E0511]: invalid monomorphization of `simd_select` intrinsic: mismatched lengths: mask length `8` != other vector length `4`
- --> $DIR/generic-select.rs:40:9
+ --> $DIR/generic-select.rs:39:9
|
LL | simd_select(m8, x, x);
| ^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `simd_select` intrinsic: mask element type is `u32`, expected `i_`
- --> $DIR/generic-select.rs:43:9
+ --> $DIR/generic-select.rs:42:9
|
LL | simd_select(x, x, x);
| ^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `simd_select` intrinsic: mask element type is `f32`, expected `i_`
- --> $DIR/generic-select.rs:46:9
+ --> $DIR/generic-select.rs:45:9
|
LL | simd_select(z, z, z);
| ^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `simd_select` intrinsic: expected SIMD argument type, found non-SIMD `u32`
- --> $DIR/generic-select.rs:49:9
+ --> $DIR/generic-select.rs:48:9
|
LL | simd_select(m4, 0u32, 1u32);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: mismatched lengths: mask length `16` != other vector length `4`
- --> $DIR/generic-select.rs:52:9
+error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: invalid bitmask `u16`, expected `u8` or `[u8; 1]`
+ --> $DIR/generic-select.rs:51:9
|
LL | simd_select_bitmask(0u16, x, x);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: expected SIMD argument type, found non-SIMD `u32`
- --> $DIR/generic-select.rs:55:9
+ --> $DIR/generic-select.rs:54:9
|
LL | simd_select_bitmask(0u8, 1u32, 2u32);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: `f32` is not an integral type
- --> $DIR/generic-select.rs:58:9
+error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: invalid bitmask `f32`, expected `u8` or `[u8; 1]`
+ --> $DIR/generic-select.rs:57:9
|
LL | simd_select_bitmask(0.0f32, x, x);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: `&str` is not an integral type
- --> $DIR/generic-select.rs:61:9
+error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: invalid bitmask `&str`, expected `u8` or `[u8; 1]`
+ --> $DIR/generic-select.rs:60:9
|
LL | simd_select_bitmask("x", x, x);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
--- /dev/null
+//run-pass
+//ignore-endian-big behavior of simd_select_bitmask is endian-specific
+#![feature(repr_simd, platform_intrinsics)]
+
+extern "platform-intrinsic" {
+ fn simd_bitmask<T, U>(v: T) -> U;
+ fn simd_select_bitmask<T, U>(m: T, a: U, b: U) -> U;
+}
+
+#[derive(Copy, Clone)]
+#[repr(simd)]
+struct Simd<T, const N: usize>([T; N]);
+
+fn main() {
+ unsafe {
+ let v = Simd::<i8, 4>([-1, 0, -1, 0]);
+ let i: u8 = simd_bitmask(v);
+ let a: [u8; 1] = simd_bitmask(v);
+
+ assert_eq!(i, 0b0101);
+ assert_eq!(a, [0b0101]);
+
+ let v = Simd::<i8, 16>([0, 0, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, -1, 0]);
+ let i: u16 = simd_bitmask(v);
+ let a: [u8; 2] = simd_bitmask(v);
+
+ assert_eq!(i, 0b0101000000001100);
+ assert_eq!(a, [0b1100, 0b01010000]);
+ }
+
+ unsafe {
+ let a = Simd::<i32, 8>([0, 1, 2, 3, 4, 5, 6, 7]);
+ let b = Simd::<i32, 8>([8, 9, 10, 11, 12, 13, 14, 15]);
+ let e = [0, 9, 2, 11, 12, 13, 14, 15];
+
+ let r = simd_select_bitmask(0b0101u8, a, b);
+ assert_eq!(r.0, e);
+
+ let r = simd_select_bitmask([0b0101u8], a, b);
+ assert_eq!(r.0, e);
+
+ let a = Simd::<i32, 16>([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
+ let b = Simd::<i32, 16>([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
+ let e = [16, 17, 2, 3, 20, 21, 22, 23, 24, 25, 26, 27, 12, 29, 14, 31];
+
+ let r = simd_select_bitmask(0b0101000000001100u16, a, b);
+ assert_eq!(r.0, e);
+
+ let r = simd_select_bitmask([0b1100u8, 0b01010000u8], a, b);
+ assert_eq!(r.0, e);
+ }
+}
projects / rust.git / commitdiff