1 use crate::simd::intrinsics;
2 use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
4 /// Constructs a new SIMD vector by copying elements from selected lanes in other vectors.
6 /// When swizzling one vector, lanes are selected by a `const` array of `usize`,
9 /// When swizzling two vectors, lanes are selected by a `const` array of [`Which`],
10 /// like [`Swizzle2`].
14 /// With a single SIMD vector, the const array specifies lane indices in that vector:
16 /// # #![feature(portable_simd)]
17 /// # use core::simd::{u32x2, u32x4, simd_swizzle};
18 /// let v = u32x4::from_array([10, 11, 12, 13]);
20 /// // Keeping the same size
21 /// let r: u32x4 = simd_swizzle!(v, [3, 0, 1, 2]);
22 /// assert_eq!(r.to_array(), [13, 10, 11, 12]);
24 /// // Changing the number of lanes
25 /// let r: u32x2 = simd_swizzle!(v, [3, 1]);
26 /// assert_eq!(r.to_array(), [13, 11]);
29 /// With two input SIMD vectors, the const array uses `Which` to specify the source of each index:
31 /// # #![feature(portable_simd)]
32 /// # use core::simd::{u32x2, u32x4, simd_swizzle, Which};
33 /// use Which::{First, Second};
34 /// let a = u32x4::from_array([0, 1, 2, 3]);
35 /// let b = u32x4::from_array([4, 5, 6, 7]);
37 /// // Keeping the same size
38 /// let r: u32x4 = simd_swizzle!(a, b, [First(0), First(1), Second(2), Second(3)]);
39 /// assert_eq!(r.to_array(), [0, 1, 6, 7]);
41 /// // Changing the number of lanes
42 /// let r: u32x2 = simd_swizzle!(a, b, [First(0), Second(0)]);
43 /// assert_eq!(r.to_array(), [0, 4]);
45 #[allow(unused_macros)]
46 pub macro simd_swizzle {
48 $vector:expr, $index:expr $(,)?
51 use $crate::simd::Swizzle;
53 impl<const LANES: usize> Swizzle<LANES, {$index.len()}> for Impl {
54 const INDEX: [usize; {$index.len()}] = $index;
56 Impl::swizzle($vector)
60 $first:expr, $second:expr, $index:expr $(,)?
63 use $crate::simd::{Which, Swizzle2};
65 impl<const LANES: usize> Swizzle2<LANES, {$index.len()}> for Impl {
66 const INDEX: [Which; {$index.len()}] = $index;
68 Impl::swizzle2($first, $second)
73 /// Specifies a lane index into one of two SIMD vectors.
75 /// This is an input type for [Swizzle2] and helper macros like [simd_swizzle].
76 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
78 /// Index of a lane in the first input SIMD vector.
80 /// Index of a lane in the second input SIMD vector.
84 /// Create a vector from the elements of another vector.
85 pub trait Swizzle<const INPUT_LANES: usize, const OUTPUT_LANES: usize> {
86 /// Map from the lanes of the input vector to the output vector.
87 const INDEX: [usize; OUTPUT_LANES];
89 /// Create a new vector from the lanes of `vector`.
91 /// Lane `i` of the output is `vector[Self::INDEX[i]]`.
93 #[must_use = "method returns a new vector and does not mutate the original inputs"]
94 fn swizzle<T>(vector: Simd<T, INPUT_LANES>) -> Simd<T, OUTPUT_LANES>
97 LaneCount<INPUT_LANES>: SupportedLaneCount,
98 LaneCount<OUTPUT_LANES>: SupportedLaneCount,
100 // Safety: `vector` is a vector, and `INDEX_IMPL` is a const array of u32.
101 unsafe { intrinsics::simd_shuffle(vector, vector, Self::INDEX_IMPL) }
105 /// Create a vector from the elements of two other vectors.
106 pub trait Swizzle2<const INPUT_LANES: usize, const OUTPUT_LANES: usize> {
107 /// Map from the lanes of the input vectors to the output vector
108 const INDEX: [Which; OUTPUT_LANES];
110 /// Create a new vector from the lanes of `first` and `second`.
112 /// Lane `i` is `first[j]` when `Self::INDEX[i]` is `First(j)`, or `second[j]` when it is
115 #[must_use = "method returns a new vector and does not mutate the original inputs"]
117 first: Simd<T, INPUT_LANES>,
118 second: Simd<T, INPUT_LANES>,
119 ) -> Simd<T, OUTPUT_LANES>
122 LaneCount<INPUT_LANES>: SupportedLaneCount,
123 LaneCount<OUTPUT_LANES>: SupportedLaneCount,
125 // Safety: `first` and `second` are vectors, and `INDEX_IMPL` is a const array of u32.
126 unsafe { intrinsics::simd_shuffle(first, second, Self::INDEX_IMPL) }
130 /// The `simd_shuffle` intrinsic expects `u32`, so do error checking and conversion here.
131 /// This trait hides `INDEX_IMPL` from the public API.
132 trait SwizzleImpl<const INPUT_LANES: usize, const OUTPUT_LANES: usize> {
133 const INDEX_IMPL: [u32; OUTPUT_LANES];
136 impl<T, const INPUT_LANES: usize, const OUTPUT_LANES: usize> SwizzleImpl<INPUT_LANES, OUTPUT_LANES>
139 T: Swizzle<INPUT_LANES, OUTPUT_LANES> + ?Sized,
141 const INDEX_IMPL: [u32; OUTPUT_LANES] = {
142 let mut output = [0; OUTPUT_LANES];
144 while i < OUTPUT_LANES {
145 let index = Self::INDEX[i];
146 assert!(index as u32 as usize == index);
147 assert!(index < INPUT_LANES, "source lane exceeds input lane count",);
148 output[i] = index as u32;
155 /// The `simd_shuffle` intrinsic expects `u32`, so do error checking and conversion here.
156 /// This trait hides `INDEX_IMPL` from the public API.
157 trait Swizzle2Impl<const INPUT_LANES: usize, const OUTPUT_LANES: usize> {
158 const INDEX_IMPL: [u32; OUTPUT_LANES];
161 impl<T, const INPUT_LANES: usize, const OUTPUT_LANES: usize> Swizzle2Impl<INPUT_LANES, OUTPUT_LANES>
164 T: Swizzle2<INPUT_LANES, OUTPUT_LANES> + ?Sized,
166 const INDEX_IMPL: [u32; OUTPUT_LANES] = {
167 let mut output = [0; OUTPUT_LANES];
169 while i < OUTPUT_LANES {
170 let (offset, index) = match Self::INDEX[i] {
171 Which::First(index) => (false, index),
172 Which::Second(index) => (true, index),
174 assert!(index < INPUT_LANES, "source lane exceeds input lane count",);
176 // lanes are indexed by the first vector, then second vector
177 let index = if offset { index + INPUT_LANES } else { index };
178 assert!(index as u32 as usize == index);
179 output[i] = index as u32;
186 impl<T, const LANES: usize> Simd<T, LANES>
189 LaneCount<LANES>: SupportedLaneCount,
191 /// Reverse the order of the lanes in the vector.
193 #[must_use = "method returns a new vector and does not mutate the original inputs"]
194 pub fn reverse(self) -> Self {
195 const fn reverse_index<const LANES: usize>() -> [usize; LANES] {
196 let mut index = [0; LANES];
199 index[i] = LANES - i - 1;
207 impl<const LANES: usize> Swizzle<LANES, LANES> for Reverse {
208 const INDEX: [usize; LANES] = reverse_index::<LANES>();
211 Reverse::swizzle(self)
214 /// Rotates the vector such that the first `OFFSET` elements of the slice move to the end
215 /// while the last `LANES - OFFSET` elements move to the front. After calling `rotate_lanes_left`,
216 /// the element previously in lane `OFFSET` will become the first element in the slice.
218 #[must_use = "method returns a new vector and does not mutate the original inputs"]
219 pub fn rotate_lanes_left<const OFFSET: usize>(self) -> Self {
220 const fn rotate_index<const OFFSET: usize, const LANES: usize>() -> [usize; LANES] {
221 let offset = OFFSET % LANES;
222 let mut index = [0; LANES];
225 index[i] = (i + offset) % LANES;
231 struct Rotate<const OFFSET: usize>;
233 impl<const OFFSET: usize, const LANES: usize> Swizzle<LANES, LANES> for Rotate<OFFSET> {
234 const INDEX: [usize; LANES] = rotate_index::<OFFSET, LANES>();
237 Rotate::<OFFSET>::swizzle(self)
240 /// Rotates the vector such that the first `LANES - OFFSET` elements of the vector move to
241 /// the end while the last `OFFSET` elements move to the front. After calling `rotate_lanes_right`,
242 /// the element previously at index `LANES - OFFSET` will become the first element in the slice.
244 #[must_use = "method returns a new vector and does not mutate the original inputs"]
245 pub fn rotate_lanes_right<const OFFSET: usize>(self) -> Self {
246 const fn rotate_index<const OFFSET: usize, const LANES: usize>() -> [usize; LANES] {
247 let offset = LANES - OFFSET % LANES;
248 let mut index = [0; LANES];
251 index[i] = (i + offset) % LANES;
257 struct Rotate<const OFFSET: usize>;
259 impl<const OFFSET: usize, const LANES: usize> Swizzle<LANES, LANES> for Rotate<OFFSET> {
260 const INDEX: [usize; LANES] = rotate_index::<OFFSET, LANES>();
263 Rotate::<OFFSET>::swizzle(self)
266 /// Interleave two vectors.
268 /// Produces two vectors with lanes taken alternately from `self` and `other`.
270 /// The first result contains the first `LANES / 2` lanes from `self` and `other`,
271 /// alternating, starting with the first lane of `self`.
273 /// The second result contains the last `LANES / 2` lanes from `self` and `other`,
274 /// alternating, starting with the lane `LANES / 2` from the start of `self`.
277 /// #![feature(portable_simd)]
278 /// # use core::simd::Simd;
279 /// let a = Simd::from_array([0, 1, 2, 3]);
280 /// let b = Simd::from_array([4, 5, 6, 7]);
281 /// let (x, y) = a.interleave(b);
282 /// assert_eq!(x.to_array(), [0, 4, 1, 5]);
283 /// assert_eq!(y.to_array(), [2, 6, 3, 7]);
286 #[must_use = "method returns a new vector and does not mutate the original inputs"]
287 pub fn interleave(self, other: Self) -> (Self, Self) {
288 const fn lo<const LANES: usize>() -> [Which; LANES] {
289 let mut idx = [Which::First(0); LANES];
293 idx[i] = if i % 2 == 0 {
296 Which::Second(offset)
302 const fn hi<const LANES: usize>() -> [Which; LANES] {
303 let mut idx = [Which::First(0); LANES];
306 let offset = (LANES + i) / 2;
307 idx[i] = if i % 2 == 0 {
310 Which::Second(offset)
320 impl<const LANES: usize> Swizzle2<LANES, LANES> for Lo {
321 const INDEX: [Which; LANES] = lo::<LANES>();
324 impl<const LANES: usize> Swizzle2<LANES, LANES> for Hi {
325 const INDEX: [Which; LANES] = hi::<LANES>();
328 (Lo::swizzle2(self, other), Hi::swizzle2(self, other))
331 /// Deinterleave two vectors.
333 /// The first result takes every other lane of `self` and then `other`, starting with
336 /// The second result takes every other lane of `self` and then `other`, starting with
340 /// #![feature(portable_simd)]
341 /// # use core::simd::Simd;
342 /// let a = Simd::from_array([0, 4, 1, 5]);
343 /// let b = Simd::from_array([2, 6, 3, 7]);
344 /// let (x, y) = a.deinterleave(b);
345 /// assert_eq!(x.to_array(), [0, 1, 2, 3]);
346 /// assert_eq!(y.to_array(), [4, 5, 6, 7]);
349 #[must_use = "method returns a new vector and does not mutate the original inputs"]
350 pub fn deinterleave(self, other: Self) -> (Self, Self) {
351 const fn even<const LANES: usize>() -> [Which; LANES] {
352 let mut idx = [Which::First(0); LANES];
354 while i < LANES / 2 {
355 idx[i] = Which::First(2 * i);
356 idx[i + LANES / 2] = Which::Second(2 * i);
361 const fn odd<const LANES: usize>() -> [Which; LANES] {
362 let mut idx = [Which::First(0); LANES];
364 while i < LANES / 2 {
365 idx[i] = Which::First(2 * i + 1);
366 idx[i + LANES / 2] = Which::Second(2 * i + 1);
375 impl<const LANES: usize> Swizzle2<LANES, LANES> for Even {
376 const INDEX: [Which; LANES] = even::<LANES>();
379 impl<const LANES: usize> Swizzle2<LANES, LANES> for Odd {
380 const INDEX: [Which; LANES] = odd::<LANES>();
383 (Even::swizzle2(self, other), Odd::swizzle2(self, other))