src/base.rs

   1 use rustc_middle::ty::adjustment::PointerCast;
   2 use rustc_index::vec::IndexVec;
   3
   4 use crate::prelude::*;
   5
   6 pub(crate) fn trans_fn<'tcx, B: Backend + 'static>(
   7     cx: &mut crate::CodegenCx<'tcx, B>,
   8     instance: Instance<'tcx>,
   9     linkage: Linkage,
  10 ) {
  11     let tcx = cx.tcx;
  12
  13     let mir = tcx.instance_mir(instance.def);
  14
  15     // Declare function
  16     let (name, sig) = get_function_name_and_sig(tcx, cx.module.isa().triple(), instance, false);
  17     let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
  18
  19     cx.cached_context.clear();
  20
  21     // Make the FunctionBuilder
  22     let mut func_ctx = FunctionBuilderContext::new();
  23     let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new());
  24     func.name = ExternalName::user(0, func_id.as_u32());
  25     func.signature = sig;
  26     func.collect_debug_info();
  27
  28     let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx);
  29
  30     // Predefine blocks
  31     let start_block = bcx.create_block();
  32     let block_map: IndexVec<BasicBlock, Block> = (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
  33
  34     // Make FunctionCx
  35     let pointer_type = cx.module.target_config().pointer_type();
  36     let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
  37
  38     let mut fx = FunctionCx {
  39         cx,
  40         tcx,
  41         pointer_type,
  42
  43         instance,
  44         mir,
  45
  46         bcx,
  47         block_map,
  48         local_map: FxHashMap::with_capacity_and_hasher(mir.local_decls.len(), Default::default()),
  49         caller_location: None, // set by `codegen_fn_prelude`
  50         cold_blocks: EntitySet::new(),
  51
  52         clif_comments,
  53         source_info_set: indexmap::IndexSet::new(),
  54         next_ssa_var: 0,
  55
  56         inline_asm_index: 0,
  57     };
  58
  59     let arg_uninhabited = fx.mir.args_iter().any(|arg| fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
  60
  61     if arg_uninhabited {
  62         fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
  63         fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
  64         crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument");
  65     } else {
  66         tcx.sess.time("codegen clif ir", || {
  67             tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
  68             codegen_fn_content(&mut fx);
  69         });
  70     }
  71
  72     // Recover all necessary data from fx, before accessing func will prevent future access to it.
  73     let instance = fx.instance;
  74     let mut clif_comments = fx.clif_comments;
  75     let source_info_set = fx.source_info_set;
  76     let local_map = fx.local_map;
  77     let cold_blocks = fx.cold_blocks;
  78
  79     // Store function in context
  80     let context = &mut cx.cached_context;
  81     context.func = func;
  82
  83     crate::pretty_clif::write_clif_file(
  84         tcx,
  85         "unopt",
  86         None,
  87         instance,
  88         &context,
  89         &clif_comments,
  90     );
  91
  92     // Verify function
  93     verify_func(tcx, &clif_comments, &context.func);
  94
  95     // Perform rust specific optimizations
  96     tcx.sess.time("optimize clif ir", || {
  97         crate::optimize::optimize_function(tcx, instance, context, &cold_blocks, &mut clif_comments);
  98     });
  99
 100     // If the return block is not reachable, then the SSA builder may have inserted a `iconst.i128`
 101     // instruction, which doesn't have an encoding.
 102     context.compute_cfg();
 103     context.compute_domtree();
 104     context.eliminate_unreachable_code(cx.module.isa()).unwrap();
 105
 106     // Define function
 107     let module = &mut cx.module;
 108     tcx.sess.time(
 109         "define function",
 110         || module.define_function(
 111             func_id,
 112             context,
 113             &mut cranelift_codegen::binemit::NullTrapSink {},
 114         ).unwrap(),
 115     );
 116
 117     // Write optimized function to file for debugging
 118     crate::pretty_clif::write_clif_file(
 119         tcx,
 120         "opt",
 121         Some(cx.module.isa()),
 122         instance,
 123         &context,
 124         &clif_comments,
 125     );
 126
 127     // Define debuginfo for function
 128     let isa = cx.module.isa();
 129     let debug_context = &mut cx.debug_context;
 130     let unwind_context = &mut cx.unwind_context;
 131     tcx.sess.time("generate debug info", || {
 132         if let Some(debug_context) = debug_context {
 133             debug_context.define_function(instance, func_id, &name, isa, context, &source_info_set, local_map);
 134         }
 135         unwind_context.add_function(func_id, &context, isa);
 136     });
 137
 138     // Clear context to make it usable for the next function
 139     context.clear();
 140 }
 141
 142 pub(crate) fn verify_func(tcx: TyCtxt<'_>, writer: &crate::pretty_clif::CommentWriter, func: &Function) {
 143     tcx.sess.time("verify clif ir", || {
 144         let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder());
 145         match cranelift_codegen::verify_function(&func, &flags) {
 146             Ok(_) => {}
 147             Err(err) => {
 148                 tcx.sess.err(&format!("{:?}", err));
 149                 let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error(
 150                     &func,
 151                     None,
 152                     Some(Box::new(writer)),
 153                     err,
 154                 );
 155                 tcx.sess
 156                     .fatal(&format!("cranelift verify error:\n{}", pretty_error));
 157             }
 158         }
 159     });
 160 }
 161
 162 fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Backend>) {
 163     crate::constant::check_constants(fx);
 164
 165     for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
 166         let block = fx.get_block(bb);
 167         fx.bcx.switch_to_block(block);
 168
 169         if bb_data.is_cleanup {
 170             // Unwinding after panicking is not supported
 171             continue;
 172
 173             // FIXME once unwinding is supported uncomment next lines
 174             // // Unwinding is unlikely to happen, so mark cleanup block's as cold.
 175             // fx.cold_blocks.insert(block);
 176         }
 177
 178         fx.bcx.ins().nop();
 179         for stmt in &bb_data.statements {
 180             fx.set_debug_loc(stmt.source_info);
 181             trans_stmt(fx, block, stmt);
 182         }
 183
 184         #[cfg(debug_assertions)]
 185         {
 186             let mut terminator_head = "\n".to_string();
 187             bb_data
 188                 .terminator()
 189                 .kind
 190                 .fmt_head(&mut terminator_head)
 191                 .unwrap();
 192             let inst = fx.bcx.func.layout.last_inst(block).unwrap();
 193             fx.add_comment(inst, terminator_head);
 194         }
 195
 196         fx.set_debug_loc(bb_data.terminator().source_info);
 197
 198         match &bb_data.terminator().kind {
 199             TerminatorKind::Goto { target } => {
 200                 if let TerminatorKind::Return = fx.mir[*target].terminator().kind {
 201                     let mut can_immediately_return = true;
 202                     for stmt in &fx.mir[*target].statements {
 203                         if let StatementKind::StorageDead(_) = stmt.kind {
 204                         } else {
 205                             // FIXME Can sometimes happen, see rust-lang/rust#70531
 206                             can_immediately_return = false;
 207                             break;
 208                         }
 209                     }
 210
 211                     if can_immediately_return {
 212                         crate::abi::codegen_return(fx);
 213                         continue;
 214                     }
 215                 }
 216
 217                 let block = fx.get_block(*target);
 218                 fx.bcx.ins().jump(block, &[]);
 219             }
 220             TerminatorKind::Return => {
 221                 crate::abi::codegen_return(fx);
 222             }
 223             TerminatorKind::Assert {
 224                 cond,
 225                 expected,
 226                 msg,
 227                 target,
 228                 cleanup: _,
 229             } => {
 230                 if !fx.tcx.sess.overflow_checks() {
 231                     if let mir::AssertKind::OverflowNeg(_) = *msg {
 232                         let target = fx.get_block(*target);
 233                         fx.bcx.ins().jump(target, &[]);
 234                         continue;
 235                     }
 236                 }
 237                 let cond = trans_operand(fx, cond).load_scalar(fx);
 238
 239                 let target = fx.get_block(*target);
 240                 let failure = fx.bcx.create_block();
 241                 fx.cold_blocks.insert(failure);
 242
 243                 if *expected {
 244                     fx.bcx.ins().brz(cond, failure, &[]);
 245                 } else {
 246                     fx.bcx.ins().brnz(cond, failure, &[]);
 247                 };
 248                 fx.bcx.ins().jump(target, &[]);
 249
 250                 fx.bcx.switch_to_block(failure);
 251
 252                 let location = fx.get_caller_location(bb_data.terminator().source_info.span).load_scalar(fx);
 253
 254                 let args;
 255                 let lang_item = match msg {
 256                     AssertKind::BoundsCheck { ref len, ref index } => {
 257                         let len = trans_operand(fx, len).load_scalar(fx);
 258                         let index = trans_operand(fx, index).load_scalar(fx);
 259                         args = [index, len, location];
 260                         rustc_hir::lang_items::PanicBoundsCheckFnLangItem
 261                     }
 262                     _ => {
 263                         let msg_str = msg.description();
 264                         let msg_ptr = fx.anonymous_str("assert", msg_str);
 265                         let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
 266                         args = [msg_ptr, msg_len, location];
 267                         rustc_hir::lang_items::PanicFnLangItem
 268                     }
 269                 };
 270
 271                 let def_id = fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| {
 272                     fx.tcx.sess.span_fatal(bb_data.terminator().source_info.span, &s)
 273                 });
 274
 275                 let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
 276                 let symbol_name = fx.tcx.symbol_name(instance).name;
 277
 278                 fx.lib_call(&*symbol_name, vec![fx.pointer_type, fx.pointer_type, fx.pointer_type], vec![], &args);
 279
 280                 crate::trap::trap_unreachable(fx, "panic lang item returned");
 281             }
 282
 283             TerminatorKind::SwitchInt {
 284                 discr,
 285                 switch_ty: _,
 286                 values,
 287                 targets,
 288             } => {
 289                 let discr = trans_operand(fx, discr).load_scalar(fx);
 290                 let mut switch = ::cranelift_frontend::Switch::new();
 291                 for (i, value) in values.iter().enumerate() {
 292                     let block = fx.get_block(targets[i]);
 293                     switch.set_entry(*value, block);
 294                 }
 295                 let otherwise_block = fx.get_block(targets[targets.len() - 1]);
 296                 switch.emit(&mut fx.bcx, discr, otherwise_block);
 297             }
 298             TerminatorKind::Call {
 299                 func,
 300                 args,
 301                 destination,
 302                 fn_span,
 303                 cleanup: _,
 304                 from_hir_call: _,
 305             } => {
 306                 fx.tcx.sess.time("codegen call", || crate::abi::codegen_terminator_call(
 307                     fx,
 308                     *fn_span,
 309                     block,
 310                     func,
 311                     args,
 312                     *destination,
 313                 ));
 314             }
 315             TerminatorKind::InlineAsm {
 316                 template,
 317                 operands,
 318                 options,
 319                 destination,
 320                 line_spans: _,
 321             } => {
 322                 crate::inline_asm::codegen_inline_asm(
 323                     fx,
 324                     bb_data.terminator().source_info.span,
 325                     template,
 326                     operands,
 327                     *options,
 328                 );
 329
 330                 match *destination {
 331                     Some(destination) => {
 332                         let destination_block = fx.get_block(destination);
 333                         fx.bcx.ins().jump(destination_block, &[]);
 334                     }
 335                     None => {
 336                         crate::trap::trap_unreachable(fx, "[corruption] Returned from noreturn inline asm");
 337                     }
 338                 }
 339             }
 340             TerminatorKind::Resume | TerminatorKind::Abort => {
 341                 trap_unreachable(fx, "[corruption] Unwinding bb reached.");
 342             }
 343             TerminatorKind::Unreachable => {
 344                 trap_unreachable(fx, "[corruption] Hit unreachable code.");
 345             }
 346             TerminatorKind::Yield { .. }
 347             | TerminatorKind::FalseEdge { .. }
 348             | TerminatorKind::FalseUnwind { .. }
 349             | TerminatorKind::DropAndReplace { .. }
 350             | TerminatorKind::GeneratorDrop => {
 351                 bug!("shouldn't exist at trans {:?}", bb_data.terminator());
 352             }
 353             TerminatorKind::Drop {
 354                 place,
 355                 target,
 356                 unwind: _,
 357             } => {
 358                 let drop_place = trans_place(fx, *place);
 359                 crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place);
 360
 361                 let target_block = fx.get_block(*target);
 362                 fx.bcx.ins().jump(target_block, &[]);
 363             }
 364         };
 365     }
 366
 367     fx.bcx.seal_all_blocks();
 368     fx.bcx.finalize();
 369 }
 370
 371 fn trans_stmt<'tcx>(
 372     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 373     #[allow(unused_variables)]
 374     cur_block: Block,
 375     stmt: &Statement<'tcx>,
 376 ) {
 377     let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt));
 378
 379     fx.set_debug_loc(stmt.source_info);
 380
 381     #[cfg(false_debug_assertions)]
 382     match &stmt.kind {
 383         StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
 384         _ => {
 385             let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
 386             fx.add_comment(inst, format!("{:?}", stmt));
 387         }
 388     }
 389
 390     match &stmt.kind {
 391         StatementKind::SetDiscriminant {
 392             place,
 393             variant_index,
 394         } => {
 395             let place = trans_place(fx, **place);
 396             crate::discriminant::codegen_set_discriminant(fx, place, *variant_index);
 397         }
 398         StatementKind::Assign(to_place_and_rval) => {
 399             let lval = trans_place(fx, to_place_and_rval.0);
 400             let dest_layout = lval.layout();
 401             match &to_place_and_rval.1 {
 402                 Rvalue::Use(operand) => {
 403                     let val = trans_operand(fx, operand);
 404                     lval.write_cvalue(fx, val);
 405                 }
 406                 Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
 407                     let place = trans_place(fx, *place);
 408                     place.write_place_ref(fx, lval);
 409                 }
 410                 Rvalue::ThreadLocalRef(def_id) => {
 411                     let val = crate::constant::codegen_tls_ref(fx, *def_id, lval.layout());
 412                     lval.write_cvalue(fx, val);
 413                 }
 414                 Rvalue::BinaryOp(bin_op, lhs, rhs) => {
 415                     let lhs = trans_operand(fx, lhs);
 416                     let rhs = trans_operand(fx, rhs);
 417
 418                     let res = crate::num::codegen_binop(fx, *bin_op, lhs, rhs);
 419                     lval.write_cvalue(fx, res);
 420                 }
 421                 Rvalue::CheckedBinaryOp(bin_op, lhs, rhs) => {
 422                     let lhs = trans_operand(fx, lhs);
 423                     let rhs = trans_operand(fx, rhs);
 424
 425                     let res = if !fx.tcx.sess.overflow_checks() {
 426                         let val =
 427                             crate::num::trans_int_binop(fx, *bin_op, lhs, rhs).load_scalar(fx);
 428                         let is_overflow = fx.bcx.ins().iconst(types::I8, 0);
 429                         CValue::by_val_pair(val, is_overflow, lval.layout())
 430                     } else {
 431                         crate::num::trans_checked_int_binop(fx, *bin_op, lhs, rhs)
 432                     };
 433
 434                     lval.write_cvalue(fx, res);
 435                 }
 436                 Rvalue::UnaryOp(un_op, operand) => {
 437                     let operand = trans_operand(fx, operand);
 438                     let layout = operand.layout();
 439                     let val = operand.load_scalar(fx);
 440                     let res = match un_op {
 441                         UnOp::Not => {
 442                             match layout.ty.kind {
 443                                 ty::Bool => {
 444                                     let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0);
 445                                     CValue::by_val(fx.bcx.ins().bint(types::I8, res), layout)
 446                                 }
 447                                 ty::Uint(_) | ty::Int(_) => {
 448                                     CValue::by_val(fx.bcx.ins().bnot(val), layout)
 449                                 }
 450                                 _ => unreachable!("un op Not for {:?}", layout.ty),
 451                             }
 452                         }
 453                         UnOp::Neg => match layout.ty.kind {
 454                             ty::Int(IntTy::I128) => {
 455                                 // FIXME remove this case once ineg.i128 works
 456                                 let zero = CValue::const_val(fx, layout, 0);
 457                                 crate::num::trans_int_binop(fx, BinOp::Sub, zero, operand)
 458                             }
 459                             ty::Int(_) => {
 460                                 CValue::by_val(fx.bcx.ins().ineg(val), layout)
 461                             }
 462                             ty::Float(_) => {
 463                                 CValue::by_val(fx.bcx.ins().fneg(val), layout)
 464                             }
 465                             _ => unreachable!("un op Neg for {:?}", layout.ty),
 466                         },
 467                     };
 468                     lval.write_cvalue(fx, res);
 469                 }
 470                 Rvalue::Cast(CastKind::Pointer(PointerCast::ReifyFnPointer), operand, to_ty) => {
 471                     let from_ty = fx.monomorphize(&operand.ty(&fx.mir.local_decls, fx.tcx));
 472                     let to_layout = fx.layout_of(fx.monomorphize(to_ty));
 473                     match from_ty.kind {
 474                         ty::FnDef(def_id, substs) => {
 475                             let func_ref = fx.get_function_ref(
 476                                 Instance::resolve_for_fn_ptr(fx.tcx, ParamEnv::reveal_all(), def_id, substs)
 477                                     .unwrap()
 478                                     .polymorphize(fx.tcx),
 479                             );
 480                             let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref);
 481                             lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout));
 482                         }
 483                         _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty),
 484                     }
 485                 }
 486                 Rvalue::Cast(CastKind::Pointer(PointerCast::UnsafeFnPointer), operand, to_ty)
 487                 | Rvalue::Cast(CastKind::Pointer(PointerCast::MutToConstPointer), operand, to_ty)
 488                 | Rvalue::Cast(CastKind::Pointer(PointerCast::ArrayToPointer), operand, to_ty) => {
 489                     let to_layout = fx.layout_of(fx.monomorphize(to_ty));
 490                     let operand = trans_operand(fx, operand);
 491                     lval.write_cvalue(fx, operand.cast_pointer_to(to_layout));
 492                 }
 493                 Rvalue::Cast(CastKind::Misc, operand, to_ty) => {
 494                     let operand = trans_operand(fx, operand);
 495                     let from_ty = operand.layout().ty;
 496                     let to_ty = fx.monomorphize(to_ty);
 497
 498                     fn is_fat_ptr<'tcx>(
 499                         fx: &FunctionCx<'_, 'tcx, impl Backend>,
 500                         ty: Ty<'tcx>,
 501                     ) -> bool {
 502                         ty.builtin_deref(true)
 503                             .map(
 504                                 |ty::TypeAndMut {
 505                                      ty: pointee_ty,
 506                                      mutbl: _,
 507                                  }| has_ptr_meta(fx.tcx, pointee_ty),
 508                             )
 509                             .unwrap_or(false)
 510                     }
 511
 512                     if is_fat_ptr(fx, from_ty) {
 513                         if is_fat_ptr(fx, to_ty) {
 514                             // fat-ptr -> fat-ptr
 515                             lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout));
 516                         } else {
 517                             // fat-ptr -> thin-ptr
 518                             let (ptr, _extra) = operand.load_scalar_pair(fx);
 519                             lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout))
 520                         }
 521                     } else if let ty::Adt(adt_def, _substs) = from_ty.kind {
 522                         // enum -> discriminant value
 523                         assert!(adt_def.is_enum());
 524                         match to_ty.kind {
 525                             ty::Uint(_) | ty::Int(_) => {}
 526                             _ => unreachable!("cast adt {} -> {}", from_ty, to_ty),
 527                         }
 528
 529                         use rustc_target::abi::{TagEncoding, Int, Variants};
 530
 531                         match &operand.layout().variants {
 532                             Variants::Single { index } => {
 533                                 let discr = operand.layout().ty.discriminant_for_variant(fx.tcx, *index).unwrap();
 534                                 let discr = if discr.ty.is_signed() {
 535                                     rustc_middle::mir::interpret::sign_extend(discr.val, fx.layout_of(discr.ty).size)
 536                                 } else {
 537                                     discr.val
 538                                 };
 539
 540                                 let discr = CValue::const_val(fx, fx.layout_of(to_ty), discr);
 541                                 lval.write_cvalue(fx, discr);
 542                             }
 543                             Variants::Multiple {
 544                                 tag,
 545                                 tag_field,
 546                                 tag_encoding: TagEncoding::Direct,
 547                                 variants: _,
 548                             } => {
 549                                 let cast_to = fx.clif_type(dest_layout.ty).unwrap();
 550
 551                                 // Read the tag/niche-encoded discriminant from memory.
 552                                 let encoded_discr = operand.value_field(fx, mir::Field::new(*tag_field));
 553                                 let encoded_discr = encoded_discr.load_scalar(fx);
 554
 555                                 // Decode the discriminant (specifically if it's niche-encoded).
 556                                 let signed = match tag.value {
 557                                     Int(_, signed) => signed,
 558                                     _ => false,
 559                                 };
 560                                 let val = clif_intcast(fx, encoded_discr, cast_to, signed);
 561                                 let val = CValue::by_val(val, dest_layout);
 562                                 lval.write_cvalue(fx, val);
 563                             }
 564                             Variants::Multiple { ..} => unreachable!(),
 565                         }
 566                     } else {
 567                         let to_clif_ty = fx.clif_type(to_ty).unwrap();
 568                         let from = operand.load_scalar(fx);
 569
 570                         let res = clif_int_or_float_cast(
 571                             fx,
 572                             from,
 573                             type_sign(from_ty),
 574                             to_clif_ty,
 575                             type_sign(to_ty),
 576                         );
 577                         lval.write_cvalue(fx, CValue::by_val(res, dest_layout));
 578                     }
 579                 }
 580                 Rvalue::Cast(CastKind::Pointer(PointerCast::ClosureFnPointer(_)), operand, _to_ty) => {
 581                     let operand = trans_operand(fx, operand);
 582                     match operand.layout().ty.kind {
 583                         ty::Closure(def_id, substs) => {
 584                             let instance = Instance::resolve_closure(
 585                                 fx.tcx,
 586                                 def_id,
 587                                 substs,
 588                                 ty::ClosureKind::FnOnce,
 589                             ).polymorphize(fx.tcx);
 590                             let func_ref = fx.get_function_ref(instance);
 591                             let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref);
 592                             lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout()));
 593                         }
 594                         _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty),
 595                     }
 596                 }
 597                 Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), operand, _to_ty) => {
 598                     let operand = trans_operand(fx, operand);
 599                     operand.unsize_value(fx, lval);
 600                 }
 601                 Rvalue::Discriminant(place) => {
 602                     let place = trans_place(fx, *place);
 603                     let value = place.to_cvalue(fx);
 604                     let discr =
 605                         crate::discriminant::codegen_get_discriminant(fx, value, dest_layout);
 606                     lval.write_cvalue(fx, discr);
 607                 }
 608                 Rvalue::Repeat(operand, times) => {
 609                     let operand = trans_operand(fx, operand);
 610                     let times = fx
 611                         .monomorphize(times)
 612                         .eval(fx.tcx, ParamEnv::reveal_all())
 613                         .val
 614                         .try_to_bits(fx.tcx.data_layout.pointer_size)
 615                         .unwrap();
 616                     for i in 0..times {
 617                         let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64);
 618                         let to = lval.place_index(fx, index);
 619                         to.write_cvalue(fx, operand);
 620                     }
 621                 }
 622                 Rvalue::Len(place) => {
 623                     let place = trans_place(fx, *place);
 624                     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 625                     let len = codegen_array_len(fx, place);
 626                     lval.write_cvalue(fx, CValue::by_val(len, usize_layout));
 627                 }
 628                 Rvalue::NullaryOp(NullOp::Box, content_ty) => {
 629                     use rustc_hir::lang_items::ExchangeMallocFnLangItem;
 630
 631                     let usize_type = fx.clif_type(fx.tcx.types.usize).unwrap();
 632                     let content_ty = fx.monomorphize(content_ty);
 633                     let layout = fx.layout_of(content_ty);
 634                     let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64);
 635                     let llalign = fx
 636                         .bcx
 637                         .ins()
 638                         .iconst(usize_type, layout.align.abi.bytes() as i64);
 639                     let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty));
 640
 641                     // Allocate space:
 642                     let def_id = match fx.tcx.lang_items().require(ExchangeMallocFnLangItem) {
 643                         Ok(id) => id,
 644                         Err(s) => {
 645                             fx.tcx
 646                                 .sess
 647                                 .fatal(&format!("allocation of `{}` {}", box_layout.ty, s));
 648                         }
 649                     };
 650                     let instance = ty::Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
 651                     let func_ref = fx.get_function_ref(instance);
 652                     let call = fx.bcx.ins().call(func_ref, &[llsize, llalign]);
 653                     let ptr = fx.bcx.inst_results(call)[0];
 654                     lval.write_cvalue(fx, CValue::by_val(ptr, box_layout));
 655                 }
 656                 Rvalue::NullaryOp(NullOp::SizeOf, ty) => {
 657                     assert!(lval
 658                         .layout()
 659                         .ty
 660                         .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all()));
 661                     let ty_size = fx.layout_of(fx.monomorphize(ty)).size.bytes();
 662                     let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), ty_size.into());
 663                     lval.write_cvalue(fx, val);
 664                 }
 665                 Rvalue::Aggregate(kind, operands) => match **kind {
 666                     AggregateKind::Array(_ty) => {
 667                         for (i, operand) in operands.into_iter().enumerate() {
 668                             let operand = trans_operand(fx, operand);
 669                             let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64);
 670                             let to = lval.place_index(fx, index);
 671                             to.write_cvalue(fx, operand);
 672                         }
 673                     }
 674                     _ => unreachable!("shouldn't exist at trans {:?}", to_place_and_rval.1),
 675                 },
 676             }
 677         }
 678         StatementKind::StorageLive(_)
 679         | StatementKind::StorageDead(_)
 680         | StatementKind::Nop
 681         | StatementKind::FakeRead(..)
 682         | StatementKind::Retag { .. }
 683         | StatementKind::AscribeUserType(..) => {}
 684
 685         StatementKind::LlvmInlineAsm(asm) => {
 686             use rustc_span::symbol::Symbol;
 687             let LlvmInlineAsm {
 688                 asm,
 689                 outputs,
 690                 inputs,
 691             } = &**asm;
 692             let rustc_hir::LlvmInlineAsmInner {
 693                 asm: asm_code, // Name
 694                 outputs: output_names, // Vec<LlvmInlineAsmOutput>
 695                 inputs: input_names,   // Vec<Name>
 696                 clobbers,      // Vec<Name>
 697                 volatile,      // bool
 698                 alignstack,    // bool
 699                 dialect: _,
 700                 asm_str_style: _,
 701             } = asm;
 702             match asm_code.as_str().trim() {
 703                 "" => {
 704                     // Black box
 705                 }
 706                 "mov %rbx, %rsi\n                  cpuid\n                  xchg %rbx, %rsi" => {
 707                     assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]);
 708                     assert_eq!(output_names.len(), 4);
 709                     for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() {
 710                         assert_eq!(&output_names[i].constraint.as_str(), c);
 711                         assert!(!output_names[i].is_rw);
 712                         assert!(!output_names[i].is_indirect);
 713                     }
 714
 715                     assert_eq!(clobbers, &[]);
 716
 717                     assert!(!volatile);
 718                     assert!(!alignstack);
 719
 720                     assert_eq!(inputs.len(), 2);
 721                     let leaf = trans_operand(fx, &inputs[0].1).load_scalar(fx); // %eax
 722                     let subleaf = trans_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx
 723
 724                     let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf);
 725
 726                     assert_eq!(outputs.len(), 4);
 727                     trans_place(fx, outputs[0]).write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32)));
 728                     trans_place(fx, outputs[1]).write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
 729                     trans_place(fx, outputs[2]).write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
 730                     trans_place(fx, outputs[3]).write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
 731                 }
 732                 "xgetbv" => {
 733                     assert_eq!(input_names, &[Symbol::intern("{ecx}")]);
 734
 735                     assert_eq!(output_names.len(), 2);
 736                     for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() {
 737                         assert_eq!(&output_names[i].constraint.as_str(), c);
 738                         assert!(!output_names[i].is_rw);
 739                         assert!(!output_names[i].is_indirect);
 740                     }
 741
 742                     assert_eq!(clobbers, &[]);
 743
 744                     assert!(!volatile);
 745                     assert!(!alignstack);
 746
 747                     crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported");
 748                 }
 749                 // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
 750                 _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => {
 751                     crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
 752                 }
 753                 _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => {
 754                     crate::trap::trap_unimplemented(fx, "Alloca is not supported");
 755                 }
 756                 // Used in sys::windows::abort_internal
 757                 "int $$0x29" => {
 758                     crate::trap::trap_unimplemented(fx, "Windows abort");
 759                 }
 760                 _ => fx.tcx.sess.span_fatal(stmt.source_info.span, "Inline assembly is not supported"),
 761             }
 762         }
 763     }
 764 }
 765
 766 fn codegen_array_len<'tcx>(
 767     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 768     place: CPlace<'tcx>,
 769 ) -> Value {
 770     match place.layout().ty.kind {
 771         ty::Array(_elem_ty, len) => {
 772             let len = fx.monomorphize(&len)
 773                 .eval(fx.tcx, ParamEnv::reveal_all())
 774                 .eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
 775             fx.bcx.ins().iconst(fx.pointer_type, len)
 776         }
 777         ty::Slice(_elem_ty) => place
 778             .to_ptr_maybe_unsized()
 779             .1
 780             .expect("Length metadata for slice place"),
 781         _ => bug!("Rvalue::Len({:?})", place),
 782     }
 783 }
 784
 785 pub(crate) fn trans_place<'tcx>(
 786     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 787     place: Place<'tcx>,
 788 ) -> CPlace<'tcx> {
 789     let mut cplace = fx.get_local_place(place.local);
 790
 791     for elem in place.projection {
 792         match elem {
 793             PlaceElem::Deref => {
 794                 cplace = cplace.place_deref(fx);
 795             }
 796             PlaceElem::Field(field, _ty) => {
 797                 cplace = cplace.place_field(fx, field);
 798             }
 799             PlaceElem::Index(local) => {
 800                 let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx);
 801                 cplace = cplace.place_index(fx, index);
 802             }
 803             PlaceElem::ConstantIndex {
 804                 offset,
 805                 min_length: _,
 806                 from_end,
 807             } => {
 808                 let index = if !from_end {
 809                     fx.bcx.ins().iconst(fx.pointer_type, i64::from(offset))
 810                 } else {
 811                     let len = codegen_array_len(fx, cplace);
 812                     fx.bcx.ins().iadd_imm(len, -i64::from(offset))
 813                 };
 814                 cplace = cplace.place_index(fx, index);
 815             }
 816             PlaceElem::Subslice { from, to, from_end } => {
 817                 // These indices are generated by slice patterns.
 818                 // slice[from:-to] in Python terms.
 819
 820                 match cplace.layout().ty.kind {
 821                     ty::Array(elem_ty, _len) => {
 822                         assert!(!from_end, "array subslices are never `from_end`");
 823                         let elem_layout = fx.layout_of(elem_ty);
 824                         let ptr = cplace.to_ptr();
 825                         cplace = CPlace::for_ptr(
 826                             ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * i64::from(from)),
 827                             fx.layout_of(fx.tcx.mk_array(elem_ty, u64::from(to) - u64::from(from))),
 828                         );
 829                     }
 830                     ty::Slice(elem_ty) => {
 831                         assert!(from_end, "slice subslices should be `from_end`");
 832                         let elem_layout = fx.layout_of(elem_ty);
 833                         let (ptr, len) = cplace.to_ptr_maybe_unsized();
 834                         let len = len.unwrap();
 835                         cplace = CPlace::for_ptr_with_extra(
 836                             ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * i64::from(from)),
 837                             fx.bcx.ins().iadd_imm(len, -(i64::from(from) + i64::from(to))),
 838                             cplace.layout(),
 839                         );
 840                     }
 841                     _ => unreachable!(),
 842                 }
 843             }
 844             PlaceElem::Downcast(_adt_def, variant) => {
 845                 cplace = cplace.downcast_variant(fx, variant);
 846             }
 847         }
 848     }
 849
 850     cplace
 851 }
 852
 853 pub(crate) fn trans_operand<'tcx>(
 854     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 855     operand: &Operand<'tcx>,
 856 ) -> CValue<'tcx> {
 857     match operand {
 858         Operand::Move(place) | Operand::Copy(place) => {
 859             let cplace = trans_place(fx, *place);
 860             cplace.to_cvalue(fx)
 861         }
 862         Operand::Constant(const_) => crate::constant::trans_constant(fx, const_),
 863     }
 864 }