compiler/rustc_codegen_cranelift/src/num.rs

   1 //! Various operations on integer and floating-point numbers
   2
   3 use crate::prelude::*;
   4
   5 pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
   6     use BinOp::*;
   7     use IntCC::*;
   8     Some(match bin_op {
   9         Eq => Equal,
  10         Lt => {
  11             if signed {
  12                 SignedLessThan
  13             } else {
  14                 UnsignedLessThan
  15             }
  16         }
  17         Le => {
  18             if signed {
  19                 SignedLessThanOrEqual
  20             } else {
  21                 UnsignedLessThanOrEqual
  22             }
  23         }
  24         Ne => NotEqual,
  25         Ge => {
  26             if signed {
  27                 SignedGreaterThanOrEqual
  28             } else {
  29                 UnsignedGreaterThanOrEqual
  30             }
  31         }
  32         Gt => {
  33             if signed {
  34                 SignedGreaterThan
  35             } else {
  36                 UnsignedGreaterThan
  37             }
  38         }
  39         _ => return None,
  40     })
  41 }
  42
  43 fn codegen_compare_bin_op<'tcx>(
  44     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
  45     bin_op: BinOp,
  46     signed: bool,
  47     lhs: Value,
  48     rhs: Value,
  49 ) -> CValue<'tcx> {
  50     let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
  51     let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
  52     let val = fx.bcx.ins().bint(types::I8, val);
  53     CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
  54 }
  55
  56 pub(crate) fn codegen_binop<'tcx>(
  57     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
  58     bin_op: BinOp,
  59     in_lhs: CValue<'tcx>,
  60     in_rhs: CValue<'tcx>,
  61 ) -> CValue<'tcx> {
  62     match bin_op {
  63         BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
  64             match in_lhs.layout().ty.kind() {
  65                 ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => {
  66                     let signed = type_sign(in_lhs.layout().ty);
  67                     let lhs = in_lhs.load_scalar(fx);
  68                     let rhs = in_rhs.load_scalar(fx);
  69
  70                     let (lhs, rhs) = if (bin_op == BinOp::Eq || bin_op == BinOp::Ne)
  71                         && (in_lhs.layout().ty.kind() == fx.tcx.types.i8.kind()
  72                             || in_lhs.layout().ty.kind() == fx.tcx.types.i16.kind())
  73                     {
  74                         // FIXME(CraneStation/cranelift#896) icmp_imm.i8/i16 with eq/ne for signed ints is implemented wrong.
  75                         (
  76                             fx.bcx.ins().sextend(types::I32, lhs),
  77                             fx.bcx.ins().sextend(types::I32, rhs),
  78                         )
  79                     } else {
  80                         (lhs, rhs)
  81                     };
  82
  83                     return codegen_compare_bin_op(fx, bin_op, signed, lhs, rhs);
  84                 }
  85                 _ => {}
  86             }
  87         }
  88         _ => {}
  89     }
  90
  91     match in_lhs.layout().ty.kind() {
  92         ty::Bool => crate::num::codegen_bool_binop(fx, bin_op, in_lhs, in_rhs),
  93         ty::Uint(_) | ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs),
  94         ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs),
  95         ty::RawPtr(..) | ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs),
  96         _ => unreachable!(
  97             "{:?}({:?}, {:?})",
  98             bin_op,
  99             in_lhs.layout().ty,
 100             in_rhs.layout().ty
 101         ),
 102     }
 103 }
 104
 105 pub(crate) fn codegen_bool_binop<'tcx>(
 106     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 107     bin_op: BinOp,
 108     in_lhs: CValue<'tcx>,
 109     in_rhs: CValue<'tcx>,
 110 ) -> CValue<'tcx> {
 111     let lhs = in_lhs.load_scalar(fx);
 112     let rhs = in_rhs.load_scalar(fx);
 113
 114     let b = fx.bcx.ins();
 115     let res = match bin_op {
 116         BinOp::BitXor => b.bxor(lhs, rhs),
 117         BinOp::BitAnd => b.band(lhs, rhs),
 118         BinOp::BitOr => b.bor(lhs, rhs),
 119         // Compare binops handles by `codegen_binop`.
 120         _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
 121     };
 122
 123     CValue::by_val(res, fx.layout_of(fx.tcx.types.bool))
 124 }
 125
 126 pub(crate) fn codegen_int_binop<'tcx>(
 127     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 128     bin_op: BinOp,
 129     in_lhs: CValue<'tcx>,
 130     in_rhs: CValue<'tcx>,
 131 ) -> CValue<'tcx> {
 132     if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
 133         assert_eq!(
 134             in_lhs.layout().ty,
 135             in_rhs.layout().ty,
 136             "int binop requires lhs and rhs of same type"
 137         );
 138     }
 139
 140     if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, false, in_lhs, in_rhs) {
 141         return res;
 142     }
 143
 144     let signed = type_sign(in_lhs.layout().ty);
 145
 146     let lhs = in_lhs.load_scalar(fx);
 147     let rhs = in_rhs.load_scalar(fx);
 148
 149     let b = fx.bcx.ins();
 150     let val = match bin_op {
 151         BinOp::Add => b.iadd(lhs, rhs),
 152         BinOp::Sub => b.isub(lhs, rhs),
 153         BinOp::Mul => b.imul(lhs, rhs),
 154         BinOp::Div => {
 155             if signed {
 156                 b.sdiv(lhs, rhs)
 157             } else {
 158                 b.udiv(lhs, rhs)
 159             }
 160         }
 161         BinOp::Rem => {
 162             if signed {
 163                 b.srem(lhs, rhs)
 164             } else {
 165                 b.urem(lhs, rhs)
 166             }
 167         }
 168         BinOp::BitXor => b.bxor(lhs, rhs),
 169         BinOp::BitAnd => b.band(lhs, rhs),
 170         BinOp::BitOr => b.bor(lhs, rhs),
 171         BinOp::Shl => {
 172             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
 173             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
 174             let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
 175             fx.bcx.ins().ishl(lhs, actual_shift)
 176         }
 177         BinOp::Shr => {
 178             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
 179             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
 180             let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
 181             if signed {
 182                 fx.bcx.ins().sshr(lhs, actual_shift)
 183             } else {
 184                 fx.bcx.ins().ushr(lhs, actual_shift)
 185             }
 186         }
 187         // Compare binops handles by `codegen_binop`.
 188         _ => unreachable!(
 189             "{:?}({:?}, {:?})",
 190             bin_op,
 191             in_lhs.layout().ty,
 192             in_rhs.layout().ty
 193         ),
 194     };
 195
 196     CValue::by_val(val, in_lhs.layout())
 197 }
 198
 199 pub(crate) fn codegen_checked_int_binop<'tcx>(
 200     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 201     bin_op: BinOp,
 202     in_lhs: CValue<'tcx>,
 203     in_rhs: CValue<'tcx>,
 204 ) -> CValue<'tcx> {
 205     if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
 206         assert_eq!(
 207             in_lhs.layout().ty,
 208             in_rhs.layout().ty,
 209             "checked int binop requires lhs and rhs of same type"
 210         );
 211     }
 212
 213     let lhs = in_lhs.load_scalar(fx);
 214     let rhs = in_rhs.load_scalar(fx);
 215
 216     if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, true, in_lhs, in_rhs) {
 217         return res;
 218     }
 219
 220     let signed = type_sign(in_lhs.layout().ty);
 221
 222     let (res, has_overflow) = match bin_op {
 223         BinOp::Add => {
 224             /*let (val, c_out) = fx.bcx.ins().iadd_cout(lhs, rhs);
 225             (val, c_out)*/
 226             // FIXME(CraneStation/cranelift#849) legalize iadd_cout for i8 and i16
 227             let val = fx.bcx.ins().iadd(lhs, rhs);
 228             let has_overflow = if !signed {
 229                 fx.bcx.ins().icmp(IntCC::UnsignedLessThan, val, lhs)
 230             } else {
 231                 let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
 232                 let slt = fx.bcx.ins().icmp(IntCC::SignedLessThan, val, lhs);
 233                 fx.bcx.ins().bxor(rhs_is_negative, slt)
 234             };
 235             (val, has_overflow)
 236         }
 237         BinOp::Sub => {
 238             /*let (val, b_out) = fx.bcx.ins().isub_bout(lhs, rhs);
 239             (val, b_out)*/
 240             // FIXME(CraneStation/cranelift#849) legalize isub_bout for i8 and i16
 241             let val = fx.bcx.ins().isub(lhs, rhs);
 242             let has_overflow = if !signed {
 243                 fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, val, lhs)
 244             } else {
 245                 let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
 246                 let sgt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, val, lhs);
 247                 fx.bcx.ins().bxor(rhs_is_negative, sgt)
 248             };
 249             (val, has_overflow)
 250         }
 251         BinOp::Mul => {
 252             let ty = fx.bcx.func.dfg.value_type(lhs);
 253             match ty {
 254                 types::I8 | types::I16 | types::I32 if !signed => {
 255                     let lhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), lhs);
 256                     let rhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), rhs);
 257                     let val = fx.bcx.ins().imul(lhs, rhs);
 258                     let has_overflow = fx.bcx.ins().icmp_imm(
 259                         IntCC::UnsignedGreaterThan,
 260                         val,
 261                         (1 << ty.bits()) - 1,
 262                     );
 263                     let val = fx.bcx.ins().ireduce(ty, val);
 264                     (val, has_overflow)
 265                 }
 266                 types::I8 | types::I16 | types::I32 if signed => {
 267                     let lhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), lhs);
 268                     let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs);
 269                     let val = fx.bcx.ins().imul(lhs, rhs);
 270                     let has_underflow =
 271                         fx.bcx
 272                             .ins()
 273                             .icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
 274                     let has_overflow = fx.bcx.ins().icmp_imm(
 275                         IntCC::SignedGreaterThan,
 276                         val,
 277                         (1 << (ty.bits() - 1)) - 1,
 278                     );
 279                     let val = fx.bcx.ins().ireduce(ty, val);
 280                     (val, fx.bcx.ins().bor(has_underflow, has_overflow))
 281                 }
 282                 types::I64 => {
 283                     let val = fx.bcx.ins().imul(lhs, rhs);
 284                     let has_overflow = if !signed {
 285                         let val_hi = fx.bcx.ins().umulhi(lhs, rhs);
 286                         fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0)
 287                     } else {
 288                         let val_hi = fx.bcx.ins().smulhi(lhs, rhs);
 289                         let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0);
 290                         let not_all_ones = fx.bcx.ins().icmp_imm(
 291                             IntCC::NotEqual,
 292                             val_hi,
 293                             u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64,
 294                         );
 295                         fx.bcx.ins().band(not_all_zero, not_all_ones)
 296                     };
 297                     (val, has_overflow)
 298                 }
 299                 types::I128 => {
 300                     unreachable!("i128 should have been handled by codegen_i128::maybe_codegen")
 301                 }
 302                 _ => unreachable!("invalid non-integer type {}", ty),
 303             }
 304         }
 305         BinOp::Shl => {
 306             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
 307             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
 308             let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
 309             let val = fx.bcx.ins().ishl(lhs, actual_shift);
 310             let ty = fx.bcx.func.dfg.value_type(val);
 311             let max_shift = i64::from(ty.bits()) - 1;
 312             let has_overflow = fx
 313                 .bcx
 314                 .ins()
 315                 .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
 316             (val, has_overflow)
 317         }
 318         BinOp::Shr => {
 319             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
 320             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
 321             let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
 322             let val = if !signed {
 323                 fx.bcx.ins().ushr(lhs, actual_shift)
 324             } else {
 325                 fx.bcx.ins().sshr(lhs, actual_shift)
 326             };
 327             let ty = fx.bcx.func.dfg.value_type(val);
 328             let max_shift = i64::from(ty.bits()) - 1;
 329             let has_overflow = fx
 330                 .bcx
 331                 .ins()
 332                 .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
 333             (val, has_overflow)
 334         }
 335         _ => bug!(
 336             "binop {:?} on checked int/uint lhs: {:?} rhs: {:?}",
 337             bin_op,
 338             in_lhs,
 339             in_rhs
 340         ),
 341     };
 342
 343     let has_overflow = fx.bcx.ins().bint(types::I8, has_overflow);
 344
 345     // FIXME directly write to result place instead
 346     let out_place = CPlace::new_stack_slot(
 347         fx,
 348         fx.layout_of(
 349             fx.tcx
 350                 .mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()),
 351         ),
 352     );
 353     let out_layout = out_place.layout();
 354     out_place.write_cvalue(fx, CValue::by_val_pair(res, has_overflow, out_layout));
 355
 356     out_place.to_cvalue(fx)
 357 }
 358
 359 pub(crate) fn codegen_float_binop<'tcx>(
 360     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 361     bin_op: BinOp,
 362     in_lhs: CValue<'tcx>,
 363     in_rhs: CValue<'tcx>,
 364 ) -> CValue<'tcx> {
 365     assert_eq!(in_lhs.layout().ty, in_rhs.layout().ty);
 366
 367     let lhs = in_lhs.load_scalar(fx);
 368     let rhs = in_rhs.load_scalar(fx);
 369
 370     let b = fx.bcx.ins();
 371     let res = match bin_op {
 372         BinOp::Add => b.fadd(lhs, rhs),
 373         BinOp::Sub => b.fsub(lhs, rhs),
 374         BinOp::Mul => b.fmul(lhs, rhs),
 375         BinOp::Div => b.fdiv(lhs, rhs),
 376         BinOp::Rem => {
 377             let name = match in_lhs.layout().ty.kind() {
 378                 ty::Float(FloatTy::F32) => "fmodf",
 379                 ty::Float(FloatTy::F64) => "fmod",
 380                 _ => bug!(),
 381             };
 382             return fx.easy_call(name, &[in_lhs, in_rhs], in_lhs.layout().ty);
 383         }
 384         BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
 385             let fltcc = match bin_op {
 386                 BinOp::Eq => FloatCC::Equal,
 387                 BinOp::Lt => FloatCC::LessThan,
 388                 BinOp::Le => FloatCC::LessThanOrEqual,
 389                 BinOp::Ne => FloatCC::NotEqual,
 390                 BinOp::Ge => FloatCC::GreaterThanOrEqual,
 391                 BinOp::Gt => FloatCC::GreaterThan,
 392                 _ => unreachable!(),
 393             };
 394             let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs);
 395             let val = fx.bcx.ins().bint(types::I8, val);
 396             return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool));
 397         }
 398         _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
 399     };
 400
 401     CValue::by_val(res, in_lhs.layout())
 402 }
 403
 404 pub(crate) fn codegen_ptr_binop<'tcx>(
 405     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 406     bin_op: BinOp,
 407     in_lhs: CValue<'tcx>,
 408     in_rhs: CValue<'tcx>,
 409 ) -> CValue<'tcx> {
 410     let is_thin_ptr = in_lhs
 411         .layout()
 412         .ty
 413         .builtin_deref(true)
 414         .map(|TypeAndMut { ty, mutbl: _ }| !has_ptr_meta(fx.tcx, ty))
 415         .unwrap_or(true);
 416
 417     if is_thin_ptr {
 418         match bin_op {
 419             BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
 420                 let lhs = in_lhs.load_scalar(fx);
 421                 let rhs = in_rhs.load_scalar(fx);
 422
 423                 return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs);
 424             }
 425             BinOp::Offset => {
 426                 let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
 427                 let (base, offset) = (in_lhs, in_rhs.load_scalar(fx));
 428                 let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 429                 let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
 430                 let base_val = base.load_scalar(fx);
 431                 let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 432                 return CValue::by_val(res, base.layout());
 433             }
 434             _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
 435         };
 436     } else {
 437         let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
 438         let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);
 439
 440         let res = match bin_op {
 441             BinOp::Eq => {
 442                 let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
 443                 let extra_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_extra, rhs_extra);
 444                 fx.bcx.ins().band(ptr_eq, extra_eq)
 445             }
 446             BinOp::Ne => {
 447                 let ptr_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_ptr, rhs_ptr);
 448                 let extra_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_extra, rhs_extra);
 449                 fx.bcx.ins().bor(ptr_ne, extra_ne)
 450             }
 451             BinOp::Lt | BinOp::Le | BinOp::Ge | BinOp::Gt => {
 452                 let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
 453
 454                 let ptr_cmp =
 455                     fx.bcx
 456                         .ins()
 457                         .icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
 458                 let extra_cmp = fx.bcx.ins().icmp(
 459                     bin_op_to_intcc(bin_op, false).unwrap(),
 460                     lhs_extra,
 461                     rhs_extra,
 462                 );
 463
 464                 fx.bcx.ins().select(ptr_eq, extra_cmp, ptr_cmp)
 465             }
 466             _ => panic!("bin_op {:?} on ptr", bin_op),
 467         };
 468
 469         CValue::by_val(
 470             fx.bcx.ins().bint(types::I8, res),
 471             fx.layout_of(fx.tcx.types.bool),
 472         )
 473     }
 474 }