9 // Vectors of pointers are not for public use at the current time.
12 use crate::simd::intrinsics;
13 use crate::simd::{LaneCount, Mask, MaskElement, SupportedLaneCount};
15 /// A SIMD vector of `LANES` elements of type `T`.
17 pub struct Simd<T, const LANES: usize>([T; LANES])
20 LaneCount<LANES>: SupportedLaneCount;
22 impl<T, const LANES: usize> Simd<T, LANES>
24 LaneCount<LANES>: SupportedLaneCount,
27 /// Number of lanes in this vector.
28 pub const LANES: usize = LANES;
30 /// Get the number of lanes in this vector.
31 pub const fn lanes(&self) -> usize {
35 /// Construct a SIMD vector by setting all lanes to the given value.
36 pub const fn splat(value: T) -> Self {
40 /// Returns an array reference containing the entire SIMD vector.
41 pub const fn as_array(&self) -> &[T; LANES] {
45 /// Returns a mutable array reference containing the entire SIMD vector.
46 pub fn as_mut_array(&mut self) -> &mut [T; LANES] {
50 /// Converts an array to a SIMD vector.
51 pub const fn from_array(array: [T; LANES]) -> Self {
55 /// Converts a SIMD vector to an array.
56 pub const fn to_array(self) -> [T; LANES] {
60 /// Converts a slice to a SIMD vector containing `slice[..LANES]`
62 /// `from_slice` will panic if the slice's `len` is less than the vector's `Simd::LANES`.
64 pub const fn from_slice(slice: &[T]) -> Self {
67 "slice length must be at least the number of lanes"
69 let mut array = [slice[0]; LANES];
78 /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
79 /// If an index is out-of-bounds, the lane is instead selected from the `or` vector.
83 /// # #![feature(portable_simd)]
84 /// # #[cfg(feature = "std")] use core_simd::Simd;
85 /// # #[cfg(not(feature = "std"))] use core::simd::Simd;
86 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
87 /// let idxs = Simd::from_array([9, 3, 0, 5]);
88 /// let alt = Simd::from_array([-5, -4, -3, -2]);
90 /// let result = Simd::gather_or(&vec, idxs, alt); // Note the lane that is out-of-bounds.
91 /// assert_eq!(result, Simd::from_array([-5, 13, 10, 15]));
95 pub fn gather_or(slice: &[T], idxs: Simd<usize, LANES>, or: Self) -> Self {
96 Self::gather_select(slice, Mask::splat(true), idxs, or)
99 /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
100 /// If an index is out-of-bounds, the lane is set to the default value for the type.
104 /// # #![feature(portable_simd)]
105 /// # #[cfg(feature = "std")] use core_simd::Simd;
106 /// # #[cfg(not(feature = "std"))] use core::simd::Simd;
107 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
108 /// let idxs = Simd::from_array([9, 3, 0, 5]);
110 /// let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds.
111 /// assert_eq!(result, Simd::from_array([0, 13, 10, 15]));
115 pub fn gather_or_default(slice: &[T], idxs: Simd<usize, LANES>) -> Self
119 Self::gather_or(slice, idxs, Self::splat(T::default()))
122 /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
123 /// The mask `enable`s all `true` lanes and disables all `false` lanes.
124 /// If an index is disabled or is out-of-bounds, the lane is selected from the `or` vector.
128 /// # #![feature(portable_simd)]
129 /// # #[cfg(feature = "std")] use core_simd::{Simd, Mask};
130 /// # #[cfg(not(feature = "std"))] use core::simd::{Simd, Mask};
131 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
132 /// let idxs = Simd::from_array([9, 3, 0, 5]);
133 /// let alt = Simd::from_array([-5, -4, -3, -2]);
134 /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane.
136 /// let result = Simd::gather_select(&vec, enable, idxs, alt); // Note the lane that is out-of-bounds.
137 /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2]));
141 pub fn gather_select(
143 enable: Mask<isize, LANES>,
144 idxs: Simd<usize, LANES>,
147 let enable: Mask<isize, LANES> = enable & idxs.lanes_lt(Simd::splat(slice.len()));
148 // SAFETY: We have masked-off out-of-bounds lanes.
149 unsafe { Self::gather_select_unchecked(slice, enable, idxs, or) }
152 /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
153 /// The mask `enable`s all `true` lanes and disables all `false` lanes.
154 /// If an index is disabled, the lane is selected from the `or` vector.
158 /// Calling this function with an `enable`d out-of-bounds index is *[undefined behavior]*
159 /// even if the resulting value is not used.
163 /// # #![feature(portable_simd)]
164 /// # #[cfg(feature = "std")] use core_simd::{Simd, Mask};
165 /// # #[cfg(not(feature = "std"))] use core::simd::{Simd, Mask};
166 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
167 /// let idxs = Simd::from_array([9, 3, 0, 5]);
168 /// let alt = Simd::from_array([-5, -4, -3, -2]);
169 /// let enable = Mask::from_array([true, true, true, false]); // Note the final mask lane.
170 /// // If this mask was used to gather, it would be unsound. Let's fix that.
171 /// let enable = enable & idxs.lanes_lt(Simd::splat(vec.len()));
173 /// // We have masked the OOB lane, so it's safe to gather now.
174 /// let result = unsafe { Simd::gather_select_unchecked(&vec, enable, idxs, alt) };
175 /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2]));
177 /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
180 pub unsafe fn gather_select_unchecked(
182 enable: Mask<isize, LANES>,
183 idxs: Simd<usize, LANES>,
186 let base_ptr = crate::simd::ptr::SimdConstPtr::splat(slice.as_ptr());
187 // Ferris forgive me, I have done pointer arithmetic here.
188 let ptrs = base_ptr.wrapping_add(idxs);
189 // SAFETY: The ptrs have been bounds-masked to prevent memory-unsafe reads insha'allah
190 unsafe { intrinsics::simd_gather(or, ptrs, enable.to_int()) }
193 /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`.
194 /// If two lanes in the scattered vector would write to the same index
195 /// only the last lane is guaranteed to actually be written.
199 /// # #![feature(portable_simd)]
200 /// # #[cfg(feature = "std")] use core_simd::Simd;
201 /// # #[cfg(not(feature = "std"))] use core::simd::Simd;
202 /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
203 /// let idxs = Simd::from_array([9, 3, 0, 0]);
204 /// let vals = Simd::from_array([-27, 82, -41, 124]);
206 /// vals.scatter(&mut vec, idxs); // index 0 receives two writes.
207 /// assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]);
210 pub fn scatter(self, slice: &mut [T], idxs: Simd<usize, LANES>) {
211 self.scatter_select(slice, Mask::splat(true), idxs)
214 /// Writes the values in a SIMD vector to multiple potentially discontiguous indices in `slice`.
215 /// The mask `enable`s all `true` lanes and disables all `false` lanes.
216 /// If an enabled index is out-of-bounds, the lane is not written.
217 /// If two enabled lanes in the scattered vector would write to the same index,
218 /// only the last lane is guaranteed to actually be written.
222 /// # #![feature(portable_simd)]
223 /// # #[cfg(feature = "std")] use core_simd::{Simd, Mask};
224 /// # #[cfg(not(feature = "std"))] use core::simd::{Simd, Mask};
225 /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
226 /// let idxs = Simd::from_array([9, 3, 0, 0]);
227 /// let vals = Simd::from_array([-27, 82, -41, 124]);
228 /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane.
230 /// vals.scatter_select(&mut vec, enable, idxs); // index 0's second write is masked, thus omitted.
231 /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]);
234 pub fn scatter_select(
237 enable: Mask<isize, LANES>,
238 idxs: Simd<usize, LANES>,
240 let enable: Mask<isize, LANES> = enable & idxs.lanes_lt(Simd::splat(slice.len()));
241 // SAFETY: We have masked-off out-of-bounds lanes.
242 unsafe { self.scatter_select_unchecked(slice, enable, idxs) }
245 /// Writes the values in a SIMD vector to multiple potentially discontiguous indices in `slice`.
246 /// The mask `enable`s all `true` lanes and disables all `false` lanes.
247 /// If two enabled lanes in the scattered vector would write to the same index,
248 /// only the last lane is guaranteed to actually be written.
252 /// Calling this function with an enabled out-of-bounds index is *[undefined behavior]*,
253 /// and may lead to memory corruption.
257 /// # #![feature(portable_simd)]
258 /// # #[cfg(feature = "std")] use core_simd::{Simd, Mask};
259 /// # #[cfg(not(feature = "std"))] use core::simd::{Simd, Mask};
260 /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
261 /// let idxs = Simd::from_array([9, 3, 0, 0]);
262 /// let vals = Simd::from_array([-27, 82, -41, 124]);
263 /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane.
264 /// // If this mask was used to scatter, it would be unsound. Let's fix that.
265 /// let enable = enable & idxs.lanes_lt(Simd::splat(vec.len()));
267 /// // We have masked the OOB lane, so it's safe to scatter now.
268 /// unsafe { vals.scatter_select_unchecked(&mut vec, enable, idxs); }
269 /// // index 0's second write is masked, thus was omitted.
270 /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]);
272 /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
274 pub unsafe fn scatter_select_unchecked(
277 enable: Mask<isize, LANES>,
278 idxs: Simd<usize, LANES>,
280 // SAFETY: This block works with *mut T derived from &mut 'a [T],
281 // which means it is delicate in Rust's borrowing model, circa 2021:
282 // &mut 'a [T] asserts uniqueness, so deriving &'a [T] invalidates live *mut Ts!
283 // Even though this block is largely safe methods, it must be exactly this way
284 // to prevent invalidating the raw ptrs while they're live.
285 // Thus, entering this block requires all values to use being already ready:
286 // 0. idxs we want to write to, which are used to construct the mask.
287 // 1. enable, which depends on an initial &'a [T] and the idxs.
288 // 2. actual values to scatter (self).
289 // 3. &mut [T] which will become our base ptr.
291 // Now Entering ☢️ *mut T Zone
292 let base_ptr = crate::simd::ptr::SimdMutPtr::splat(slice.as_mut_ptr());
293 // Ferris forgive me, I have done pointer arithmetic here.
294 let ptrs = base_ptr.wrapping_add(idxs);
295 // The ptrs have been bounds-masked to prevent memory-unsafe writes insha'allah
296 intrinsics::simd_scatter(self, ptrs, enable.to_int())
297 // Cleared ☢️ *mut T Zone
302 impl<T, const LANES: usize> Copy for Simd<T, LANES>
305 LaneCount<LANES>: SupportedLaneCount,
309 impl<T, const LANES: usize> Clone for Simd<T, LANES>
312 LaneCount<LANES>: SupportedLaneCount,
314 fn clone(&self) -> Self {
319 impl<T, const LANES: usize> Default for Simd<T, LANES>
321 LaneCount<LANES>: SupportedLaneCount,
322 T: SimdElement + Default,
325 fn default() -> Self {
326 Self::splat(T::default())
330 impl<T, const LANES: usize> PartialEq for Simd<T, LANES>
332 LaneCount<LANES>: SupportedLaneCount,
333 T: SimdElement + PartialEq,
336 fn eq(&self, other: &Self) -> bool {
337 // TODO use SIMD equality
338 self.to_array() == other.to_array()
342 impl<T, const LANES: usize> PartialOrd for Simd<T, LANES>
344 LaneCount<LANES>: SupportedLaneCount,
345 T: SimdElement + PartialOrd,
348 fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
349 // TODO use SIMD equality
350 self.to_array().partial_cmp(other.as_ref())
354 impl<T, const LANES: usize> Eq for Simd<T, LANES>
356 LaneCount<LANES>: SupportedLaneCount,
361 impl<T, const LANES: usize> Ord for Simd<T, LANES>
363 LaneCount<LANES>: SupportedLaneCount,
364 T: SimdElement + Ord,
367 fn cmp(&self, other: &Self) -> core::cmp::Ordering {
368 // TODO use SIMD equality
369 self.to_array().cmp(other.as_ref())
373 impl<T, const LANES: usize> core::hash::Hash for Simd<T, LANES>
375 LaneCount<LANES>: SupportedLaneCount,
376 T: SimdElement + core::hash::Hash,
379 fn hash<H>(&self, state: &mut H)
381 H: core::hash::Hasher,
383 self.as_array().hash(state)
388 impl<T, const LANES: usize> AsRef<[T; LANES]> for Simd<T, LANES>
390 LaneCount<LANES>: SupportedLaneCount,
394 fn as_ref(&self) -> &[T; LANES] {
399 impl<T, const LANES: usize> AsMut<[T; LANES]> for Simd<T, LANES>
401 LaneCount<LANES>: SupportedLaneCount,
405 fn as_mut(&mut self) -> &mut [T; LANES] {
411 impl<T, const LANES: usize> AsRef<[T]> for Simd<T, LANES>
413 LaneCount<LANES>: SupportedLaneCount,
417 fn as_ref(&self) -> &[T] {
422 impl<T, const LANES: usize> AsMut<[T]> for Simd<T, LANES>
424 LaneCount<LANES>: SupportedLaneCount,
428 fn as_mut(&mut self) -> &mut [T] {
433 // vector/array conversion
434 impl<T, const LANES: usize> From<[T; LANES]> for Simd<T, LANES>
436 LaneCount<LANES>: SupportedLaneCount,
439 fn from(array: [T; LANES]) -> Self {
444 impl<T, const LANES: usize> From<Simd<T, LANES>> for [T; LANES]
446 LaneCount<LANES>: SupportedLaneCount,
449 fn from(vector: Simd<T, LANES>) -> Self {
459 /// Marker trait for types that may be used as SIMD vector elements.
460 /// SAFETY: This trait, when implemented, asserts the compiler can monomorphize
461 /// `#[repr(simd)]` structs with the marked type as an element.
462 /// Strictly, it is valid to impl if the vector will not be miscompiled.
463 /// Practically, it is user-unfriendly to impl it if the vector won't compile,
464 /// even when no soundness guarantees are broken by allowing the user to try.
465 pub unsafe trait SimdElement: Sealed + Copy {
466 /// The mask element type corresponding to this element type.
467 type Mask: MaskElement;
470 impl Sealed for u8 {}
471 unsafe impl SimdElement for u8 {
475 impl Sealed for u16 {}
476 unsafe impl SimdElement for u16 {
480 impl Sealed for u32 {}
481 unsafe impl SimdElement for u32 {
485 impl Sealed for u64 {}
486 unsafe impl SimdElement for u64 {
490 impl Sealed for usize {}
491 unsafe impl SimdElement for usize {
495 impl Sealed for i8 {}
496 unsafe impl SimdElement for i8 {
500 impl Sealed for i16 {}
501 unsafe impl SimdElement for i16 {
505 impl Sealed for i32 {}
506 unsafe impl SimdElement for i32 {
510 impl Sealed for i64 {}
511 unsafe impl SimdElement for i64 {
515 impl Sealed for isize {}
516 unsafe impl SimdElement for isize {
520 impl Sealed for f32 {}
521 unsafe impl SimdElement for f32 {
525 impl Sealed for f64 {}
526 unsafe impl SimdElement for f64 {