From bfed3c4f0d285df97c0e9cbe342665b8b37b7a96 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 16 Mar 2022 20:33:54 -0400 Subject: [PATCH] implement simd bitmask intrinsics --- src/helpers.rs | 15 ------- src/shims/intrinsics.rs | 74 ++++++++++++++++++++++++++++++++- tests/run-pass/portable-simd.rs | 15 +++++++ 3 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src/helpers.rs b/src/helpers.rs index ba12e0a7e39..fe2f33ffd33 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -775,18 +775,3 @@ pub fn isolation_abort_error(name: &str) -> InterpResult<'static> { name, ))) } - -pub fn bool_to_simd_element(b: bool, size: Size) -> Scalar { - // SIMD uses all-1 as pattern for "true" - let val = if b { -1 } else { 0 }; - Scalar::from_int(val, size) -} - -pub fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> { - let val = elem.to_scalar()?.to_int(elem.layout.size)?; - Ok(match val { - 0 => false, - -1 => true, - _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), - }) -} diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 726e6b6b961..b704004e16c 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -1,3 +1,4 @@ +use std::convert::TryInto; use std::iter; use log::trace; @@ -5,10 +6,10 @@ use rustc_apfloat::{Float, Round}; use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf}; use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy}; -use rustc_target::abi::{Align, Integer}; +use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size}; use crate::*; -use helpers::{bool_to_simd_element, check_arg_count, simd_element_to_bool}; +use helpers::check_arg_count; pub enum AtomicOp { MirOp(mir::BinOp, bool), @@ -663,6 +664,35 @@ enum Op { this.write_immediate(*val, &dest.into())?; } } + "simd_select_bitmask" => { + let &[ref mask, ref yes, ref no] = check_arg_count(args)?; + let (yes, yes_len) = this.operand_to_simd(yes)?; + let (no, no_len) = this.operand_to_simd(no)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert!(mask.layout.ty.is_integral()); + assert_eq!(dest_len.max(8), mask.layout.size.bits()); + assert!(dest_len <= 64); + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + let mask: u64 = this + .read_scalar(mask)? + .check_init()? + .to_bits(mask.layout.size)? + .try_into() + .unwrap(); + for i in 0..dest_len { + let mask = + mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian)); + let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?; + let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + let val = if mask != 0 { yes } else { no }; + this.write_immediate(*val, &dest.into())?; + } + } #[rustfmt::skip] "simd_cast" | "simd_as" => { let &[ref op] = check_arg_count(args)?; @@ -787,6 +817,23 @@ enum Op { } } } + "simd_bitmask" => { + let &[ref op] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + + assert!(dest.layout.ty.is_integral()); + assert_eq!(op_len.max(8), dest.layout.size.bits()); + assert!(op_len <= 64); + + let mut res = 0u64; + for i in 0..op_len { + let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; + if simd_element_to_bool(op)? { + res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian); + } + } + this.write_int(res, dest)?; + } // Atomic operations "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?, @@ -1307,3 +1354,26 @@ fn fmin_op<'tcx>( FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)), }) } + +fn bool_to_simd_element(b: bool, size: Size) -> Scalar { + // SIMD uses all-1 as pattern for "true" + let val = if b { -1 } else { 0 }; + Scalar::from_int(val, size) +} + +fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> { + let val = elem.to_scalar()?.to_int(elem.layout.size)?; + Ok(match val { + 0 => false, + -1 => true, + _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), + }) +} + +fn simd_bitmask_index(idx: u64, len: u64, endianess: Endian) -> u64 { + assert!(idx < len); + match endianess { + Endian::Little => idx, + Endian::Big => len.max(8) - 1 - idx, // reverse order of bits + } +} diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index 80b0b4556c6..a74559b72be 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -187,6 +187,21 @@ fn simd_mask() { let intmask = Mask::from_int(i32x4::from_array([0, -1, 0, 0])); assert_eq!(intmask, Mask::from_array([false, true, false, false])); assert_eq!(intmask.to_array(), [false, true, false, false]); + + let values = [ + true, false, false, true, false, false, true, false, true, true, false, false, false, true, + false, true, + ]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1010001101001001); + assert_eq!(Mask::::from_bitmask(bitmask), mask); + + let values = [false, false, false, true]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1000); + assert_eq!(Mask::::from_bitmask(bitmask), mask); } fn simd_cast() { -- 2.44.0