3 use gccjit::{Function, FunctionPtrType, RValue, ToRValue};
5 use crate::{context::CodegenCx, builder::Builder};
7 pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(builder: &Builder<'a, 'gcc, 'tcx>, gcc_func: FunctionPtrType<'gcc>, mut args: Cow<'b, [RValue<'gcc>]>, func_name: &str) -> Cow<'b, [RValue<'gcc>]> {
8 // Some LLVM intrinsics do not map 1-to-1 to GCC intrinsics, so we add the missing
10 if gcc_func.get_param_count() != args.len() {
12 "__builtin_ia32_pmuldq512_mask" | "__builtin_ia32_pmuludq512_mask"
13 // FIXME(antoyo): the following intrinsics has 4 (or 5) arguments according to the doc, but is defined with 2 (or 3) arguments in library/stdarch/crates/core_arch/src/x86/avx512f.rs.
14 | "__builtin_ia32_pmaxsd512_mask" | "__builtin_ia32_pmaxsq512_mask" | "__builtin_ia32_pmaxsq256_mask"
15 | "__builtin_ia32_pmaxsq128_mask" | "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
16 | "__builtin_ia32_pmaxud512_mask" | "__builtin_ia32_pmaxuq512_mask" | "__builtin_ia32_pmaxuq256_mask"
17 | "__builtin_ia32_pmaxuq128_mask"
18 | "__builtin_ia32_pminsd512_mask" | "__builtin_ia32_pminsq512_mask" | "__builtin_ia32_pminsq256_mask"
19 | "__builtin_ia32_pminsq128_mask" | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask"
20 | "__builtin_ia32_pminud512_mask" | "__builtin_ia32_pminuq512_mask" | "__builtin_ia32_pminuq256_mask"
21 | "__builtin_ia32_pminuq128_mask" | "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask"
23 // TODO: refactor by separating those intrinsics outside of this branch.
24 let add_before_last_arg =
26 "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
27 | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask"
28 | "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => true,
31 let new_first_arg_is_zero =
33 "__builtin_ia32_pmaxuq256_mask" | "__builtin_ia32_pmaxuq128_mask"
34 | "__builtin_ia32_pminuq256_mask" | "__builtin_ia32_pminuq128_mask" => true,
39 "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => 1,
42 let mut new_args = args.to_vec();
43 let arg3_type = gcc_func.get_param_type(arg3_index);
45 if new_first_arg_is_zero {
46 let vector_type = arg3_type.dyncast_vector().expect("vector type");
47 let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
48 let num_units = vector_type.get_num_units();
49 builder.context.new_rvalue_from_vector(None, arg3_type, &vec![zero; num_units])
52 builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue()
54 if add_before_last_arg {
55 new_args.insert(new_args.len() - 1, first_arg);
58 new_args.push(first_arg);
62 "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => 2,
65 let arg4_type = gcc_func.get_param_type(arg4_index);
66 let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
67 if add_before_last_arg {
68 new_args.insert(new_args.len() - 1, minus_one);
71 new_args.push(minus_one);
73 args = new_args.into();
75 "__builtin_ia32_pternlogd512_mask" | "__builtin_ia32_pternlogd256_mask"
76 | "__builtin_ia32_pternlogd128_mask" | "__builtin_ia32_pternlogq512_mask"
77 | "__builtin_ia32_pternlogq256_mask" | "__builtin_ia32_pternlogq128_mask" => {
78 let mut new_args = args.to_vec();
79 let arg5_type = gcc_func.get_param_type(4);
80 let minus_one = builder.context.new_rvalue_from_int(arg5_type, -1);
81 new_args.push(minus_one);
82 args = new_args.into();
84 "__builtin_ia32_vfmaddps512_mask" | "__builtin_ia32_vfmaddpd512_mask" => {
85 let mut new_args = args.to_vec();
87 let mut last_arg = None;
89 last_arg = new_args.pop();
92 let arg4_type = gcc_func.get_param_type(3);
93 let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
94 new_args.push(minus_one);
97 // Both llvm.fma.v16f32 and llvm.x86.avx512.vfmadd.ps.512 maps to
98 // the same GCC intrinsic, but the former has 3 parameters and the
99 // latter has 4 so it doesn't require this additional argument.
100 let arg5_type = gcc_func.get_param_type(4);
101 new_args.push(builder.context.new_rvalue_from_int(arg5_type, 4));
104 if let Some(last_arg) = last_arg {
105 new_args.push(last_arg);
108 args = new_args.into();
110 "__builtin_ia32_addps512_mask" | "__builtin_ia32_addpd512_mask"
111 | "__builtin_ia32_subps512_mask" | "__builtin_ia32_subpd512_mask"
112 | "__builtin_ia32_mulps512_mask" | "__builtin_ia32_mulpd512_mask"
113 | "__builtin_ia32_divps512_mask" | "__builtin_ia32_divpd512_mask" => {
114 let mut new_args = args.to_vec();
115 let last_arg = new_args.pop().expect("last arg");
116 let arg3_type = gcc_func.get_param_type(2);
117 let undefined = builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue();
118 new_args.push(undefined);
119 let arg4_type = gcc_func.get_param_type(3);
120 let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
121 new_args.push(minus_one);
122 new_args.push(last_arg);
123 args = new_args.into();
125 "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
126 let mut new_args = args.to_vec();
127 let last_arg = new_args.pop().expect("last arg");
128 let arg4_type = gcc_func.get_param_type(3);
129 let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
130 new_args.push(minus_one);
131 new_args.push(last_arg);
132 args = new_args.into();
141 pub fn ignore_arg_cast(func_name: &str, index: usize, args_len: usize) -> bool {
142 // NOTE: these intrinsics have missing parameters before the last one, so ignore the
143 // last argument type check.
144 // FIXME(antoyo): find a way to refactor in order to avoid this hack.
146 "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
147 | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask" | "__builtin_ia32_sqrtps512_mask"
148 | "__builtin_ia32_sqrtpd512_mask" | "__builtin_ia32_addps512_mask" | "__builtin_ia32_addpd512_mask"
149 | "__builtin_ia32_subps512_mask" | "__builtin_ia32_subpd512_mask"
150 | "__builtin_ia32_mulps512_mask" | "__builtin_ia32_mulpd512_mask"
151 | "__builtin_ia32_divps512_mask" | "__builtin_ia32_divpd512_mask"
152 | "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
153 if index == args_len - 1 {
157 "__builtin_ia32_vfmaddps512_mask" | "__builtin_ia32_vfmaddpd512_mask" => {
158 // Since there are two LLVM intrinsics that map to each of these GCC builtins and only
159 // one of them has a missing parameter before the last one, we check the number of
160 // arguments to distinguish those cases.
161 if args_len == 4 && index == args_len - 1 {
171 #[cfg(not(feature="master"))]
172 pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
174 "llvm.x86.xgetbv" => {
175 let gcc_name = "__builtin_trap";
176 let func = cx.context.get_builtin_function(gcc_name);
177 cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
180 _ => unimplemented!("unsupported LLVM intrinsic {}", name),
184 #[cfg(feature="master")]
185 pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
186 let gcc_name = match name {
187 "llvm.x86.xgetbv" => "__builtin_ia32_xgetbv",
188 // NOTE: this doc specifies the equivalent GCC builtins: http://huonw.github.io/llvmint/llvmint/x86/index.html
189 "llvm.sqrt.v2f64" => "__builtin_ia32_sqrtpd",
190 "llvm.x86.avx512.pmul.dq.512" => "__builtin_ia32_pmuldq512_mask",
191 "llvm.x86.avx512.pmulu.dq.512" => "__builtin_ia32_pmuludq512_mask",
192 "llvm.x86.avx512.mask.pmaxs.q.256" => "__builtin_ia32_pmaxsq256_mask",
193 "llvm.x86.avx512.mask.pmaxs.q.128" => "__builtin_ia32_pmaxsq128_mask",
194 "llvm.x86.avx512.max.ps.512" => "__builtin_ia32_maxps512_mask",
195 "llvm.x86.avx512.max.pd.512" => "__builtin_ia32_maxpd512_mask",
196 "llvm.x86.avx512.mask.pmaxu.q.256" => "__builtin_ia32_pmaxuq256_mask",
197 "llvm.x86.avx512.mask.pmaxu.q.128" => "__builtin_ia32_pmaxuq128_mask",
198 "llvm.x86.avx512.mask.pmins.q.256" => "__builtin_ia32_pminsq256_mask",
199 "llvm.x86.avx512.mask.pmins.q.128" => "__builtin_ia32_pminsq128_mask",
200 "llvm.x86.avx512.min.ps.512" => "__builtin_ia32_minps512_mask",
201 "llvm.x86.avx512.min.pd.512" => "__builtin_ia32_minpd512_mask",
202 "llvm.x86.avx512.mask.pminu.q.256" => "__builtin_ia32_pminuq256_mask",
203 "llvm.x86.avx512.mask.pminu.q.128" => "__builtin_ia32_pminuq128_mask",
204 "llvm.fma.v16f32" => "__builtin_ia32_vfmaddps512_mask",
205 "llvm.fma.v8f64" => "__builtin_ia32_vfmaddpd512_mask",
206 "llvm.x86.avx512.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask",
207 "llvm.x86.avx512.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask",
208 "llvm.x86.avx512.pternlog.d.512" => "__builtin_ia32_pternlogd512_mask",
209 "llvm.x86.avx512.pternlog.d.256" => "__builtin_ia32_pternlogd256_mask",
210 "llvm.x86.avx512.pternlog.d.128" => "__builtin_ia32_pternlogd128_mask",
211 "llvm.x86.avx512.pternlog.q.512" => "__builtin_ia32_pternlogq512_mask",
212 "llvm.x86.avx512.pternlog.q.256" => "__builtin_ia32_pternlogq256_mask",
213 "llvm.x86.avx512.pternlog.q.128" => "__builtin_ia32_pternlogq128_mask",
214 "llvm.x86.avx512.add.ps.512" => "__builtin_ia32_addps512_mask",
215 "llvm.x86.avx512.add.pd.512" => "__builtin_ia32_addpd512_mask",
216 "llvm.x86.avx512.sub.ps.512" => "__builtin_ia32_subps512_mask",
217 "llvm.x86.avx512.sub.pd.512" => "__builtin_ia32_subpd512_mask",
218 "llvm.x86.avx512.mul.ps.512" => "__builtin_ia32_mulps512_mask",
219 "llvm.x86.avx512.mul.pd.512" => "__builtin_ia32_mulpd512_mask",
220 "llvm.x86.avx512.div.ps.512" => "__builtin_ia32_divps512_mask",
221 "llvm.x86.avx512.div.pd.512" => "__builtin_ia32_divpd512_mask",
222 "llvm.x86.avx512.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask",
223 "llvm.x86.avx512.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask",
225 // The above doc points to unknown builtins for the following, so override them:
226 "llvm.x86.avx2.gather.d.d" => "__builtin_ia32_gathersiv4si",
227 "llvm.x86.avx2.gather.d.d.256" => "__builtin_ia32_gathersiv8si",
228 "llvm.x86.avx2.gather.d.ps" => "__builtin_ia32_gathersiv4sf",
229 "llvm.x86.avx2.gather.d.ps.256" => "__builtin_ia32_gathersiv8sf",
230 "llvm.x86.avx2.gather.d.q" => "__builtin_ia32_gathersiv2di",
231 "llvm.x86.avx2.gather.d.q.256" => "__builtin_ia32_gathersiv4di",
232 "llvm.x86.avx2.gather.d.pd" => "__builtin_ia32_gathersiv2df",
233 "llvm.x86.avx2.gather.d.pd.256" => "__builtin_ia32_gathersiv4df",
234 "llvm.x86.avx2.gather.q.d" => "__builtin_ia32_gatherdiv4si",
235 "llvm.x86.avx2.gather.q.d.256" => "__builtin_ia32_gatherdiv4si256",
236 "llvm.x86.avx2.gather.q.ps" => "__builtin_ia32_gatherdiv4sf",
237 "llvm.x86.avx2.gather.q.ps.256" => "__builtin_ia32_gatherdiv4sf256",
238 "llvm.x86.avx2.gather.q.q" => "__builtin_ia32_gatherdiv2di",
239 "llvm.x86.avx2.gather.q.q.256" => "__builtin_ia32_gatherdiv4di",
240 "llvm.x86.avx2.gather.q.pd" => "__builtin_ia32_gatherdiv2df",
241 "llvm.x86.avx2.gather.q.pd.256" => "__builtin_ia32_gatherdiv4df",
243 // NOTE: this file is generated by https://github.com/GuillaumeGomez/llvmint/blob/master/generate_list.py
244 _ => include!("archs.rs"),
247 let func = cx.context.get_target_builtin_function(gcc_name);
248 cx.functions.borrow_mut().insert(gcc_name.to_string(), func);