compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs

   1 use std::borrow::Cow;
   2
   3 use gccjit::{Function, FunctionPtrType, RValue, ToRValue};
   4
   5 use crate::{context::CodegenCx, builder::Builder};
   6
   7 pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(builder: &Builder<'a, 'gcc, 'tcx>, gcc_func: FunctionPtrType<'gcc>, mut args: Cow<'b, [RValue<'gcc>]>, func_name: &str) -> Cow<'b, [RValue<'gcc>]> {
   8     // Some LLVM intrinsics do not map 1-to-1 to GCC intrinsics, so we add the missing
   9     // arguments here.
  10     if gcc_func.get_param_count() != args.len() {
  11         match &*func_name {
  12             "__builtin_ia32_pmuldq512_mask" | "__builtin_ia32_pmuludq512_mask"
  13                 // FIXME(antoyo): the following intrinsics has 4 (or 5) arguments according to the doc, but is defined with 2 (or 3) arguments in library/stdarch/crates/core_arch/src/x86/avx512f.rs.
  14                 | "__builtin_ia32_pmaxsd512_mask" | "__builtin_ia32_pmaxsq512_mask" | "__builtin_ia32_pmaxsq256_mask"
  15                 | "__builtin_ia32_pmaxsq128_mask" | "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
  16                 | "__builtin_ia32_pmaxud512_mask" | "__builtin_ia32_pmaxuq512_mask" | "__builtin_ia32_pmaxuq256_mask"
  17                 | "__builtin_ia32_pmaxuq128_mask"
  18                 | "__builtin_ia32_pminsd512_mask" | "__builtin_ia32_pminsq512_mask" | "__builtin_ia32_pminsq256_mask"
  19                 | "__builtin_ia32_pminsq128_mask" | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask"
  20                 | "__builtin_ia32_pminud512_mask" | "__builtin_ia32_pminuq512_mask" | "__builtin_ia32_pminuq256_mask"
  21                 | "__builtin_ia32_pminuq128_mask" | "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask"
  22                 => {
  23                     // TODO: refactor by separating those intrinsics outside of this branch.
  24                     let add_before_last_arg =
  25                         match &*func_name {
  26                             "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
  27                                 | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask"
  28                                 | "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => true,
  29                             _ => false,
  30                         };
  31                     let new_first_arg_is_zero =
  32                         match &*func_name {
  33                             "__builtin_ia32_pmaxuq256_mask" | "__builtin_ia32_pmaxuq128_mask"
  34                                 | "__builtin_ia32_pminuq256_mask" | "__builtin_ia32_pminuq128_mask" => true,
  35                             _ => false
  36                         };
  37                     let arg3_index =
  38                         match &*func_name {
  39                             "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => 1,
  40                             _ => 2,
  41                         };
  42                     let mut new_args = args.to_vec();
  43                     let arg3_type = gcc_func.get_param_type(arg3_index);
  44                     let first_arg =
  45                         if new_first_arg_is_zero {
  46                             let vector_type = arg3_type.dyncast_vector().expect("vector type");
  47                             let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
  48                             let num_units = vector_type.get_num_units();
  49                             builder.context.new_rvalue_from_vector(None, arg3_type, &vec![zero; num_units])
  50                         }
  51                         else {
  52                             builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue()
  53                         };
  54                     if add_before_last_arg {
  55                         new_args.insert(new_args.len() - 1, first_arg);
  56                     }
  57                     else {
  58                         new_args.push(first_arg);
  59                     }
  60                     let arg4_index =
  61                         match &*func_name {
  62                             "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => 2,
  63                             _ => 3,
  64                         };
  65                     let arg4_type = gcc_func.get_param_type(arg4_index);
  66                     let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
  67                     if add_before_last_arg {
  68                         new_args.insert(new_args.len() - 1, minus_one);
  69                     }
  70                     else {
  71                         new_args.push(minus_one);
  72                     }
  73                     args = new_args.into();
  74                 },
  75                 "__builtin_ia32_pternlogd512_mask" | "__builtin_ia32_pternlogd256_mask"
  76                     | "__builtin_ia32_pternlogd128_mask" | "__builtin_ia32_pternlogq512_mask"
  77                     | "__builtin_ia32_pternlogq256_mask" | "__builtin_ia32_pternlogq128_mask" => {
  78                         let mut new_args = args.to_vec();
  79                         let arg5_type = gcc_func.get_param_type(4);
  80                         let minus_one = builder.context.new_rvalue_from_int(arg5_type, -1);
  81                         new_args.push(minus_one);
  82                         args = new_args.into();
  83                     },
  84                     "__builtin_ia32_vfmaddps512_mask" | "__builtin_ia32_vfmaddpd512_mask" => {
  85                         let mut new_args = args.to_vec();
  86
  87                         let mut last_arg = None;
  88                         if args.len() == 4 {
  89                             last_arg = new_args.pop();
  90                         }
  91
  92                         let arg4_type = gcc_func.get_param_type(3);
  93                         let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
  94                         new_args.push(minus_one);
  95
  96                         if args.len() == 3 {
  97                             // Both llvm.fma.v16f32 and llvm.x86.avx512.vfmadd.ps.512 maps to
  98                             // the same GCC intrinsic, but the former has 3 parameters and the
  99                             // latter has 4 so it doesn't require this additional argument.
 100                             let arg5_type = gcc_func.get_param_type(4);
 101                             new_args.push(builder.context.new_rvalue_from_int(arg5_type, 4));
 102                         }
 103
 104                         if let Some(last_arg) = last_arg {
 105                             new_args.push(last_arg);
 106                         }
 107
 108                         args = new_args.into();
 109                     },
 110                     "__builtin_ia32_addps512_mask" | "__builtin_ia32_addpd512_mask"
 111                         | "__builtin_ia32_subps512_mask" | "__builtin_ia32_subpd512_mask"
 112                         | "__builtin_ia32_mulps512_mask" | "__builtin_ia32_mulpd512_mask"
 113                         | "__builtin_ia32_divps512_mask" | "__builtin_ia32_divpd512_mask" => {
 114                         let mut new_args = args.to_vec();
 115                         let last_arg = new_args.pop().expect("last arg");
 116                         let arg3_type = gcc_func.get_param_type(2);
 117                         let undefined = builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue();
 118                         new_args.push(undefined);
 119                         let arg4_type = gcc_func.get_param_type(3);
 120                         let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
 121                         new_args.push(minus_one);
 122                         new_args.push(last_arg);
 123                         args = new_args.into();
 124                     },
 125                     "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
 126                         let mut new_args = args.to_vec();
 127                         let last_arg = new_args.pop().expect("last arg");
 128                         let arg4_type = gcc_func.get_param_type(3);
 129                         let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
 130                         new_args.push(minus_one);
 131                         new_args.push(last_arg);
 132                         args = new_args.into();
 133                     },
 134                     _ => (),
 135         }
 136     }
 137
 138     args
 139 }
 140
 141 pub fn ignore_arg_cast(func_name: &str, index: usize, args_len: usize) -> bool {
 142     // NOTE: these intrinsics have missing parameters before the last one, so ignore the
 143     // last argument type check.
 144     // FIXME(antoyo): find a way to refactor in order to avoid this hack.
 145     match func_name {
 146         "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
 147             | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask" | "__builtin_ia32_sqrtps512_mask"
 148             | "__builtin_ia32_sqrtpd512_mask" | "__builtin_ia32_addps512_mask" | "__builtin_ia32_addpd512_mask"
 149             | "__builtin_ia32_subps512_mask" | "__builtin_ia32_subpd512_mask"
 150             | "__builtin_ia32_mulps512_mask" | "__builtin_ia32_mulpd512_mask"
 151             | "__builtin_ia32_divps512_mask" | "__builtin_ia32_divpd512_mask"
 152             | "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
 153                 if index == args_len - 1 {
 154                     return true;
 155                 }
 156             },
 157         "__builtin_ia32_vfmaddps512_mask" | "__builtin_ia32_vfmaddpd512_mask" => {
 158             // Since there are two LLVM intrinsics that map to each of these GCC builtins and only
 159             // one of them has a missing parameter before the last one, we check the number of
 160             // arguments to distinguish those cases.
 161             if args_len == 4 && index == args_len - 1 {
 162                 return true;
 163             }
 164         },
 165         _ => (),
 166     }
 167
 168     false
 169 }
 170
 171 #[cfg(not(feature="master"))]
 172 pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
 173     match name {
 174         "llvm.x86.xgetbv" => {
 175             let gcc_name = "__builtin_trap";
 176             let func = cx.context.get_builtin_function(gcc_name);
 177             cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
 178             return func;
 179         },
 180         _ => unimplemented!("unsupported LLVM intrinsic {}", name),
 181     }
 182 }
 183
 184 #[cfg(feature="master")]
 185 pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
 186     let gcc_name = match name {
 187         "llvm.x86.xgetbv" => "__builtin_ia32_xgetbv",
 188         // NOTE: this doc specifies the equivalent GCC builtins: http://huonw.github.io/llvmint/llvmint/x86/index.html
 189         "llvm.sqrt.v2f64" => "__builtin_ia32_sqrtpd",
 190         "llvm.x86.avx512.pmul.dq.512" => "__builtin_ia32_pmuldq512_mask",
 191         "llvm.x86.avx512.pmulu.dq.512" => "__builtin_ia32_pmuludq512_mask",
 192         "llvm.x86.avx512.mask.pmaxs.q.256" => "__builtin_ia32_pmaxsq256_mask",
 193         "llvm.x86.avx512.mask.pmaxs.q.128" => "__builtin_ia32_pmaxsq128_mask",
 194         "llvm.x86.avx512.max.ps.512" => "__builtin_ia32_maxps512_mask",
 195         "llvm.x86.avx512.max.pd.512" => "__builtin_ia32_maxpd512_mask",
 196         "llvm.x86.avx512.mask.pmaxu.q.256" => "__builtin_ia32_pmaxuq256_mask",
 197         "llvm.x86.avx512.mask.pmaxu.q.128" => "__builtin_ia32_pmaxuq128_mask",
 198         "llvm.x86.avx512.mask.pmins.q.256" => "__builtin_ia32_pminsq256_mask",
 199         "llvm.x86.avx512.mask.pmins.q.128" => "__builtin_ia32_pminsq128_mask",
 200         "llvm.x86.avx512.min.ps.512" => "__builtin_ia32_minps512_mask",
 201         "llvm.x86.avx512.min.pd.512" => "__builtin_ia32_minpd512_mask",
 202         "llvm.x86.avx512.mask.pminu.q.256" => "__builtin_ia32_pminuq256_mask",
 203         "llvm.x86.avx512.mask.pminu.q.128" => "__builtin_ia32_pminuq128_mask",
 204         "llvm.fma.v16f32" => "__builtin_ia32_vfmaddps512_mask",
 205         "llvm.fma.v8f64" => "__builtin_ia32_vfmaddpd512_mask",
 206         "llvm.x86.avx512.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask",
 207         "llvm.x86.avx512.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask",
 208         "llvm.x86.avx512.pternlog.d.512" => "__builtin_ia32_pternlogd512_mask",
 209         "llvm.x86.avx512.pternlog.d.256" => "__builtin_ia32_pternlogd256_mask",
 210         "llvm.x86.avx512.pternlog.d.128" => "__builtin_ia32_pternlogd128_mask",
 211         "llvm.x86.avx512.pternlog.q.512" => "__builtin_ia32_pternlogq512_mask",
 212         "llvm.x86.avx512.pternlog.q.256" => "__builtin_ia32_pternlogq256_mask",
 213         "llvm.x86.avx512.pternlog.q.128" => "__builtin_ia32_pternlogq128_mask",
 214         "llvm.x86.avx512.add.ps.512" => "__builtin_ia32_addps512_mask",
 215         "llvm.x86.avx512.add.pd.512" => "__builtin_ia32_addpd512_mask",
 216         "llvm.x86.avx512.sub.ps.512" => "__builtin_ia32_subps512_mask",
 217         "llvm.x86.avx512.sub.pd.512" => "__builtin_ia32_subpd512_mask",
 218         "llvm.x86.avx512.mul.ps.512" => "__builtin_ia32_mulps512_mask",
 219         "llvm.x86.avx512.mul.pd.512" => "__builtin_ia32_mulpd512_mask",
 220         "llvm.x86.avx512.div.ps.512" => "__builtin_ia32_divps512_mask",
 221         "llvm.x86.avx512.div.pd.512" => "__builtin_ia32_divpd512_mask",
 222         "llvm.x86.avx512.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask",
 223         "llvm.x86.avx512.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask",
 224
 225         // The above doc points to unknown builtins for the following, so override them:
 226         "llvm.x86.avx2.gather.d.d" => "__builtin_ia32_gathersiv4si",
 227         "llvm.x86.avx2.gather.d.d.256" => "__builtin_ia32_gathersiv8si",
 228         "llvm.x86.avx2.gather.d.ps" => "__builtin_ia32_gathersiv4sf",
 229         "llvm.x86.avx2.gather.d.ps.256" => "__builtin_ia32_gathersiv8sf",
 230         "llvm.x86.avx2.gather.d.q" => "__builtin_ia32_gathersiv2di",
 231         "llvm.x86.avx2.gather.d.q.256" => "__builtin_ia32_gathersiv4di",
 232         "llvm.x86.avx2.gather.d.pd" => "__builtin_ia32_gathersiv2df",
 233         "llvm.x86.avx2.gather.d.pd.256" => "__builtin_ia32_gathersiv4df",
 234         "llvm.x86.avx2.gather.q.d" => "__builtin_ia32_gatherdiv4si",
 235         "llvm.x86.avx2.gather.q.d.256" => "__builtin_ia32_gatherdiv4si256",
 236         "llvm.x86.avx2.gather.q.ps" => "__builtin_ia32_gatherdiv4sf",
 237         "llvm.x86.avx2.gather.q.ps.256" => "__builtin_ia32_gatherdiv4sf256",
 238         "llvm.x86.avx2.gather.q.q" => "__builtin_ia32_gatherdiv2di",
 239         "llvm.x86.avx2.gather.q.q.256" => "__builtin_ia32_gatherdiv4di",
 240         "llvm.x86.avx2.gather.q.pd" => "__builtin_ia32_gatherdiv2df",
 241         "llvm.x86.avx2.gather.q.pd.256" => "__builtin_ia32_gatherdiv4df",
 242         "" => "",
 243         // NOTE: this file is generated by https://github.com/GuillaumeGomez/llvmint/blob/master/generate_list.py
 244         _ => include!("archs.rs"),
 245     };
 246
 247     let func = cx.context.get_target_builtin_function(gcc_name);
 248     cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
 249     func
 250 }