1 //! Types and traits associated with masking lanes of vectors.
3 #![allow(non_camel_case_types)]
6 not(all(target_arch = "x86_64", target_feature = "avx512f")),
7 path = "masks/full_masks.rs"
10 all(target_arch = "x86_64", target_feature = "avx512f"),
11 path = "masks/bitmask.rs"
15 use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
16 use core::cmp::Ordering;
22 /// Not only does this seal the `MaskElement` trait, but these functions prevent other traits
23 /// from bleeding into the parent bounds.
25 /// For example, `eq` could be provided by requiring `MaskElement: PartialEq`, but that would
26 /// prevent us from ever removing that bound, or from implementing `MaskElement` on
27 /// non-`PartialEq` types in the future.
29 fn valid<const LANES: usize>(values: Simd<Self, LANES>) -> bool
31 LaneCount<LANES>: SupportedLaneCount,
34 fn eq(self, other: Self) -> bool;
43 /// Marker trait for types that may be used as SIMD mask elements.
44 pub unsafe trait MaskElement: SimdElement + Sealed {}
46 macro_rules! impl_element {
49 fn valid<const LANES: usize>(value: Simd<Self, LANES>) -> bool
51 LaneCount<LANES>: SupportedLaneCount,
53 (value.lanes_eq(Simd::splat(0)) | value.lanes_eq(Simd::splat(-1))).all()
56 fn eq(self, other: Self) -> bool { self == other }
58 const TRUE: Self = -1;
59 const FALSE: Self = 0;
62 unsafe impl MaskElement for $ty {}
70 impl_element! { isize }
72 /// A SIMD vector mask for `LANES` elements of width specified by `Element`.
74 /// The layout of this type is unspecified.
76 pub struct Mask<T, const LANES: usize>(mask_impl::Mask<T, LANES>)
79 LaneCount<LANES>: SupportedLaneCount;
81 impl<T, const LANES: usize> Copy for Mask<T, LANES>
84 LaneCount<LANES>: SupportedLaneCount,
88 impl<T, const LANES: usize> Clone for Mask<T, LANES>
91 LaneCount<LANES>: SupportedLaneCount,
93 fn clone(&self) -> Self {
98 impl<T, const LANES: usize> Mask<T, LANES>
101 LaneCount<LANES>: SupportedLaneCount,
103 /// Construct a mask by setting all lanes to the given value.
104 pub fn splat(value: bool) -> Self {
105 Self(mask_impl::Mask::splat(value))
108 /// Converts an array to a SIMD vector.
109 pub fn from_array(array: [bool; LANES]) -> Self {
110 let mut vector = Self::splat(false);
111 for (i, v) in array.iter().enumerate() {
117 /// Converts a SIMD vector to an array.
118 pub fn to_array(self) -> [bool; LANES] {
119 let mut array = [false; LANES];
120 for (i, v) in array.iter_mut().enumerate() {
126 /// Converts a vector of integers to a mask, where 0 represents `false` and -1
127 /// represents `true`.
130 /// All lanes must be either 0 or -1.
132 #[must_use = "method returns a new mask and does not mutate the original value"]
133 pub unsafe fn from_int_unchecked(value: Simd<T, LANES>) -> Self {
134 unsafe { Self(mask_impl::Mask::from_int_unchecked(value)) }
137 /// Converts a vector of integers to a mask, where 0 represents `false` and -1
138 /// represents `true`.
141 /// Panics if any lane is not 0 or -1.
143 #[must_use = "method returns a new mask and does not mutate the original value"]
144 pub fn from_int(value: Simd<T, LANES>) -> Self {
145 assert!(T::valid(value), "all values must be either 0 or -1",);
146 unsafe { Self::from_int_unchecked(value) }
149 /// Converts the mask to a vector of integers, where 0 represents `false` and -1
150 /// represents `true`.
152 #[must_use = "method returns a new vector and does not mutate the original value"]
153 pub fn to_int(self) -> Simd<T, LANES> {
157 /// Tests the value of the specified lane.
160 /// `lane` must be less than `LANES`.
162 #[must_use = "method returns a new bool and does not mutate the original value"]
163 pub unsafe fn test_unchecked(&self, lane: usize) -> bool {
164 unsafe { self.0.test_unchecked(lane) }
167 /// Tests the value of the specified lane.
170 /// Panics if `lane` is greater than or equal to the number of lanes in the vector.
172 #[must_use = "method returns a new bool and does not mutate the original value"]
173 pub fn test(&self, lane: usize) -> bool {
174 assert!(lane < LANES, "lane index out of range");
175 unsafe { self.test_unchecked(lane) }
178 /// Sets the value of the specified lane.
181 /// `lane` must be less than `LANES`.
183 pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) {
185 self.0.set_unchecked(lane, value);
189 /// Sets the value of the specified lane.
192 /// Panics if `lane` is greater than or equal to the number of lanes in the vector.
194 pub fn set(&mut self, lane: usize, value: bool) {
195 assert!(lane < LANES, "lane index out of range");
197 self.set_unchecked(lane, value);
201 /// Convert this mask to a bitmask, with one bit set per lane.
202 #[cfg(feature = "generic_const_exprs")]
204 #[must_use = "method returns a new array and does not mutate the original value"]
205 pub fn to_bitmask(self) -> [u8; LaneCount::<LANES>::BITMASK_LEN] {
209 /// Convert a bitmask to a mask.
210 #[cfg(feature = "generic_const_exprs")]
212 #[must_use = "method returns a new mask and does not mutate the original value"]
213 pub fn from_bitmask(bitmask: [u8; LaneCount::<LANES>::BITMASK_LEN]) -> Self {
214 Self(mask_impl::Mask::from_bitmask(bitmask))
217 /// Returns true if any lane is set, or false otherwise.
219 #[must_use = "method returns a new bool and does not mutate the original value"]
220 pub fn any(self) -> bool {
224 /// Returns true if all lanes are set, or false otherwise.
226 #[must_use = "method returns a new bool and does not mutate the original value"]
227 pub fn all(self) -> bool {
232 // vector/array conversion
233 impl<T, const LANES: usize> From<[bool; LANES]> for Mask<T, LANES>
236 LaneCount<LANES>: SupportedLaneCount,
238 fn from(array: [bool; LANES]) -> Self {
239 Self::from_array(array)
243 impl<T, const LANES: usize> From<Mask<T, LANES>> for [bool; LANES]
246 LaneCount<LANES>: SupportedLaneCount,
248 fn from(vector: Mask<T, LANES>) -> Self {
253 impl<T, const LANES: usize> Default for Mask<T, LANES>
256 LaneCount<LANES>: SupportedLaneCount,
259 #[must_use = "method returns a defaulted mask with all lanes set to false (0)"]
260 fn default() -> Self {
265 impl<T, const LANES: usize> PartialEq for Mask<T, LANES>
267 T: MaskElement + PartialEq,
268 LaneCount<LANES>: SupportedLaneCount,
271 #[must_use = "method returns a new bool and does not mutate the original value"]
272 fn eq(&self, other: &Self) -> bool {
277 impl<T, const LANES: usize> PartialOrd for Mask<T, LANES>
279 T: MaskElement + PartialOrd,
280 LaneCount<LANES>: SupportedLaneCount,
283 #[must_use = "method returns a new Ordering and does not mutate the original value"]
284 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
285 self.0.partial_cmp(&other.0)
289 impl<T, const LANES: usize> fmt::Debug for Mask<T, LANES>
291 T: MaskElement + fmt::Debug,
292 LaneCount<LANES>: SupportedLaneCount,
294 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
296 .entries((0..LANES).map(|lane| self.test(lane)))
301 impl<T, const LANES: usize> core::ops::BitAnd for Mask<T, LANES>
304 LaneCount<LANES>: SupportedLaneCount,
308 #[must_use = "method returns a new mask and does not mutate the original value"]
309 fn bitand(self, rhs: Self) -> Self {
314 impl<T, const LANES: usize> core::ops::BitAnd<bool> for Mask<T, LANES>
317 LaneCount<LANES>: SupportedLaneCount,
321 #[must_use = "method returns a new mask and does not mutate the original value"]
322 fn bitand(self, rhs: bool) -> Self {
323 self & Self::splat(rhs)
327 impl<T, const LANES: usize> core::ops::BitAnd<Mask<T, LANES>> for bool
330 LaneCount<LANES>: SupportedLaneCount,
332 type Output = Mask<T, LANES>;
334 #[must_use = "method returns a new mask and does not mutate the original value"]
335 fn bitand(self, rhs: Mask<T, LANES>) -> Mask<T, LANES> {
336 Mask::splat(self) & rhs
340 impl<T, const LANES: usize> core::ops::BitOr for Mask<T, LANES>
343 LaneCount<LANES>: SupportedLaneCount,
347 #[must_use = "method returns a new mask and does not mutate the original value"]
348 fn bitor(self, rhs: Self) -> Self {
353 impl<T, const LANES: usize> core::ops::BitOr<bool> for Mask<T, LANES>
356 LaneCount<LANES>: SupportedLaneCount,
360 #[must_use = "method returns a new mask and does not mutate the original value"]
361 fn bitor(self, rhs: bool) -> Self {
362 self | Self::splat(rhs)
366 impl<T, const LANES: usize> core::ops::BitOr<Mask<T, LANES>> for bool
369 LaneCount<LANES>: SupportedLaneCount,
371 type Output = Mask<T, LANES>;
373 #[must_use = "method returns a new mask and does not mutate the original value"]
374 fn bitor(self, rhs: Mask<T, LANES>) -> Mask<T, LANES> {
375 Mask::splat(self) | rhs
379 impl<T, const LANES: usize> core::ops::BitXor for Mask<T, LANES>
382 LaneCount<LANES>: SupportedLaneCount,
386 #[must_use = "method returns a new mask and does not mutate the original value"]
387 fn bitxor(self, rhs: Self) -> Self::Output {
392 impl<T, const LANES: usize> core::ops::BitXor<bool> for Mask<T, LANES>
395 LaneCount<LANES>: SupportedLaneCount,
399 #[must_use = "method returns a new mask and does not mutate the original value"]
400 fn bitxor(self, rhs: bool) -> Self::Output {
401 self ^ Self::splat(rhs)
405 impl<T, const LANES: usize> core::ops::BitXor<Mask<T, LANES>> for bool
408 LaneCount<LANES>: SupportedLaneCount,
410 type Output = Mask<T, LANES>;
412 #[must_use = "method returns a new mask and does not mutate the original value"]
413 fn bitxor(self, rhs: Mask<T, LANES>) -> Self::Output {
414 Mask::splat(self) ^ rhs
418 impl<T, const LANES: usize> core::ops::Not for Mask<T, LANES>
421 LaneCount<LANES>: SupportedLaneCount,
423 type Output = Mask<T, LANES>;
425 #[must_use = "method returns a new mask and does not mutate the original value"]
426 fn not(self) -> Self::Output {
431 impl<T, const LANES: usize> core::ops::BitAndAssign for Mask<T, LANES>
434 LaneCount<LANES>: SupportedLaneCount,
437 fn bitand_assign(&mut self, rhs: Self) {
438 self.0 = self.0 & rhs.0;
442 impl<T, const LANES: usize> core::ops::BitAndAssign<bool> for Mask<T, LANES>
445 LaneCount<LANES>: SupportedLaneCount,
448 fn bitand_assign(&mut self, rhs: bool) {
449 *self &= Self::splat(rhs);
453 impl<T, const LANES: usize> core::ops::BitOrAssign for Mask<T, LANES>
456 LaneCount<LANES>: SupportedLaneCount,
459 fn bitor_assign(&mut self, rhs: Self) {
460 self.0 = self.0 | rhs.0;
464 impl<T, const LANES: usize> core::ops::BitOrAssign<bool> for Mask<T, LANES>
467 LaneCount<LANES>: SupportedLaneCount,
470 fn bitor_assign(&mut self, rhs: bool) {
471 *self |= Self::splat(rhs);
475 impl<T, const LANES: usize> core::ops::BitXorAssign for Mask<T, LANES>
478 LaneCount<LANES>: SupportedLaneCount,
481 fn bitxor_assign(&mut self, rhs: Self) {
482 self.0 = self.0 ^ rhs.0;
486 impl<T, const LANES: usize> core::ops::BitXorAssign<bool> for Mask<T, LANES>
489 LaneCount<LANES>: SupportedLaneCount,
492 fn bitxor_assign(&mut self, rhs: bool) {
493 *self ^= Self::splat(rhs);
497 /// Vector of eight 8-bit masks
498 pub type mask8x8 = Mask<i8, 8>;
500 /// Vector of 16 8-bit masks
501 pub type mask8x16 = Mask<i8, 16>;
503 /// Vector of 32 8-bit masks
504 pub type mask8x32 = Mask<i8, 32>;
506 /// Vector of 16 8-bit masks
507 pub type mask8x64 = Mask<i8, 64>;
509 /// Vector of four 16-bit masks
510 pub type mask16x4 = Mask<i16, 4>;
512 /// Vector of eight 16-bit masks
513 pub type mask16x8 = Mask<i16, 8>;
515 /// Vector of 16 16-bit masks
516 pub type mask16x16 = Mask<i16, 16>;
518 /// Vector of 32 16-bit masks
519 pub type mask16x32 = Mask<i32, 32>;
521 /// Vector of two 32-bit masks
522 pub type mask32x2 = Mask<i32, 2>;
524 /// Vector of four 32-bit masks
525 pub type mask32x4 = Mask<i32, 4>;
527 /// Vector of eight 32-bit masks
528 pub type mask32x8 = Mask<i32, 8>;
530 /// Vector of 16 32-bit masks
531 pub type mask32x16 = Mask<i32, 16>;
533 /// Vector of two 64-bit masks
534 pub type mask64x2 = Mask<i64, 2>;
536 /// Vector of four 64-bit masks
537 pub type mask64x4 = Mask<i64, 4>;
539 /// Vector of eight 64-bit masks
540 pub type mask64x8 = Mask<i64, 8>;
542 /// Vector of two pointer-width masks
543 pub type masksizex2 = Mask<isize, 2>;
545 /// Vector of four pointer-width masks
546 pub type masksizex4 = Mask<isize, 4>;
548 /// Vector of eight pointer-width masks
549 pub type masksizex8 = Mask<isize, 8>;
551 macro_rules! impl_from {
552 { $from:ty => $($to:ty),* } => {
554 impl<const LANES: usize> From<Mask<$from, LANES>> for Mask<$to, LANES>
556 LaneCount<LANES>: SupportedLaneCount,
558 fn from(value: Mask<$from, LANES>) -> Self {
559 Self(value.0.convert())
565 impl_from! { i8 => i16, i32, i64, isize }
566 impl_from! { i16 => i32, i64, isize, i8 }
567 impl_from! { i32 => i64, isize, i8, i16 }
568 impl_from! { i64 => isize, i8, i16, i32 }
569 impl_from! { isize => i8, i16, i32, i64 }