compiler/rustc_mir/src/transform/promote_consts.rs

   1 //! A pass that promotes borrows of constant rvalues.
   2 //!
   3 //! The rvalues considered constant are trees of temps,
   4 //! each with exactly one initialization, and holding
   5 //! a constant value with no interior mutability.
   6 //! They are placed into a new MIR constant body in
   7 //! `promoted` and the borrow rvalue is replaced with
   8 //! a `Literal::Promoted` using the index into `promoted`
   9 //! of that constant MIR.
  10 //!
  11 //! This pass assumes that every use is dominated by an
  12 //! initialization and can otherwise silence errors, if
  13 //! move analysis runs after promotion on broken MIR.
  14
  15 use rustc_ast::LitKind;
  16 use rustc_hir as hir;
  17 use rustc_hir::def_id::DefId;
  18 use rustc_middle::mir::traversal::ReversePostorder;
  19 use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
  20 use rustc_middle::mir::*;
  21 use rustc_middle::ty::cast::CastTy;
  22 use rustc_middle::ty::subst::InternalSubsts;
  23 use rustc_middle::ty::{self, List, TyCtxt, TypeFoldable};
  24 use rustc_span::symbol::sym;
  25 use rustc_span::Span;
  26
  27 use rustc_index::vec::{Idx, IndexVec};
  28 use rustc_target::spec::abi::Abi;
  29
  30 use std::cell::Cell;
  31 use std::{cmp, iter, mem};
  32
  33 use crate::const_eval::{is_const_fn, is_unstable_const_fn};
  34 use crate::transform::check_consts::{is_lang_panic_fn, qualifs, ConstCx};
  35 use crate::transform::MirPass;
  36
  37 /// A `MirPass` for promotion.
  38 ///
  39 /// Promotion is the extraction of promotable temps into separate MIR bodies. This pass also emits
  40 /// errors when promotion of `#[rustc_args_required_const]` arguments fails.
  41 ///
  42 /// After this pass is run, `promoted_fragments` will hold the MIR body corresponding to each
  43 /// newly created `Constant`.
  44 #[derive(Default)]
  45 pub struct PromoteTemps<'tcx> {
  46     pub promoted_fragments: Cell<IndexVec<Promoted, Body<'tcx>>>,
  47 }
  48
  49 impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> {
  50     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
  51         // There's not really any point in promoting errorful MIR.
  52         //
  53         // This does not include MIR that failed const-checking, which we still try to promote.
  54         if body.return_ty().references_error() {
  55             tcx.sess.delay_span_bug(body.span, "PromoteTemps: MIR had errors");
  56             return;
  57         }
  58
  59         if body.source.promoted.is_some() {
  60             return;
  61         }
  62
  63         let mut rpo = traversal::reverse_postorder(body);
  64         let ccx = ConstCx::new(tcx, body);
  65         let (temps, all_candidates) = collect_temps_and_candidates(&ccx, &mut rpo);
  66
  67         let promotable_candidates = validate_candidates(&ccx, &temps, &all_candidates);
  68
  69         let promoted = promote_candidates(body, tcx, temps, promotable_candidates);
  70         self.promoted_fragments.set(promoted);
  71     }
  72 }
  73
  74 /// State of a temporary during collection and promotion.
  75 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
  76 pub enum TempState {
  77     /// No references to this temp.
  78     Undefined,
  79     /// One direct assignment and any number of direct uses.
  80     /// A borrow of this temp is promotable if the assigned
  81     /// value is qualified as constant.
  82     Defined { location: Location, uses: usize },
  83     /// Any other combination of assignments/uses.
  84     Unpromotable,
  85     /// This temp was part of an rvalue which got extracted
  86     /// during promotion and needs cleanup.
  87     PromotedOut,
  88 }
  89
  90 impl TempState {
  91     pub fn is_promotable(&self) -> bool {
  92         debug!("is_promotable: self={:?}", self);
  93         matches!(self, TempState::Defined { .. })
  94     }
  95 }
  96
  97 /// A "root candidate" for promotion, which will become the
  98 /// returned value in a promoted MIR, unless it's a subset
  99 /// of a larger candidate.
 100 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 101 pub enum Candidate {
 102     /// Borrow of a constant temporary, candidate for lifetime extension.
 103     Ref(Location),
 104
 105     /// Currently applied to function calls where the callee has the unstable
 106     /// `#[rustc_args_required_const]` attribute as well as the SIMD shuffle
 107     /// intrinsic. The intrinsic requires the arguments are indeed constant and
 108     /// the attribute currently provides the semantic requirement that arguments
 109     /// must be constant.
 110     Argument { bb: BasicBlock, index: usize },
 111
 112     /// `const` operand in asm!.
 113     InlineAsm { bb: BasicBlock, index: usize },
 114 }
 115
 116 impl Candidate {
 117     /// Returns `true` if we should use the "explicit" rules for promotability for this `Candidate`.
 118     fn forces_explicit_promotion(&self) -> bool {
 119         match self {
 120             Candidate::Ref(_) => false,
 121             Candidate::Argument { .. } | Candidate::InlineAsm { .. } => true,
 122         }
 123     }
 124
 125     fn source_info(&self, body: &Body<'_>) -> SourceInfo {
 126         match self {
 127             Candidate::Ref(location) => *body.source_info(*location),
 128             Candidate::Argument { bb, .. } | Candidate::InlineAsm { bb, .. } => {
 129                 *body.source_info(body.terminator_loc(*bb))
 130             }
 131         }
 132     }
 133 }
 134
 135 fn args_required_const(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Vec<usize>> {
 136     let attrs = tcx.get_attrs(def_id);
 137     let attr = attrs.iter().find(|a| tcx.sess.check_name(a, sym::rustc_args_required_const))?;
 138     let mut ret = vec![];
 139     for meta in attr.meta_item_list()? {
 140         match meta.literal()?.kind {
 141             LitKind::Int(a, _) => {
 142                 ret.push(a as usize);
 143             }
 144             _ => bug!("invalid arg index"),
 145         }
 146     }
 147     Some(ret)
 148 }
 149
 150 struct Collector<'a, 'tcx> {
 151     ccx: &'a ConstCx<'a, 'tcx>,
 152     temps: IndexVec<Local, TempState>,
 153     candidates: Vec<Candidate>,
 154 }
 155
 156 impl<'tcx> Visitor<'tcx> for Collector<'_, 'tcx> {
 157     fn visit_local(&mut self, &index: &Local, context: PlaceContext, location: Location) {
 158         debug!("visit_local: index={:?} context={:?} location={:?}", index, context, location);
 159         // We're only interested in temporaries and the return place
 160         match self.ccx.body.local_kind(index) {
 161             LocalKind::Temp | LocalKind::ReturnPointer => {}
 162             LocalKind::Arg | LocalKind::Var => return,
 163         }
 164
 165         // Ignore drops, if the temp gets promoted,
 166         // then it's constant and thus drop is noop.
 167         // Non-uses are also irrelevant.
 168         if context.is_drop() || !context.is_use() {
 169             debug!(
 170                 "visit_local: context.is_drop={:?} context.is_use={:?}",
 171                 context.is_drop(),
 172                 context.is_use(),
 173             );
 174             return;
 175         }
 176
 177         let temp = &mut self.temps[index];
 178         debug!("visit_local: temp={:?}", temp);
 179         if *temp == TempState::Undefined {
 180             match context {
 181                 PlaceContext::MutatingUse(MutatingUseContext::Store)
 182                 | PlaceContext::MutatingUse(MutatingUseContext::Call) => {
 183                     *temp = TempState::Defined { location, uses: 0 };
 184                     return;
 185                 }
 186                 _ => { /* mark as unpromotable below */ }
 187             }
 188         } else if let TempState::Defined { ref mut uses, .. } = *temp {
 189             // We always allow borrows, even mutable ones, as we need
 190             // to promote mutable borrows of some ZSTs e.g., `&mut []`.
 191             let allowed_use = match context {
 192                 PlaceContext::MutatingUse(MutatingUseContext::Borrow)
 193                 | PlaceContext::NonMutatingUse(_) => true,
 194                 PlaceContext::MutatingUse(_) | PlaceContext::NonUse(_) => false,
 195             };
 196             debug!("visit_local: allowed_use={:?}", allowed_use);
 197             if allowed_use {
 198                 *uses += 1;
 199                 return;
 200             }
 201             /* mark as unpromotable below */
 202         }
 203         *temp = TempState::Unpromotable;
 204     }
 205
 206     fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
 207         self.super_rvalue(rvalue, location);
 208
 209         match *rvalue {
 210             Rvalue::Ref(..) => {
 211                 self.candidates.push(Candidate::Ref(location));
 212             }
 213             _ => {}
 214         }
 215     }
 216
 217     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
 218         self.super_terminator(terminator, location);
 219
 220         match terminator.kind {
 221             TerminatorKind::Call { ref func, .. } => {
 222                 if let ty::FnDef(def_id, _) = *func.ty(self.ccx.body, self.ccx.tcx).kind() {
 223                     let fn_sig = self.ccx.tcx.fn_sig(def_id);
 224                     if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = fn_sig.abi() {
 225                         let name = self.ccx.tcx.item_name(def_id);
 226                         // FIXME(eddyb) use `#[rustc_args_required_const(2)]` for shuffles.
 227                         if name.as_str().starts_with("simd_shuffle") {
 228                             self.candidates
 229                                 .push(Candidate::Argument { bb: location.block, index: 2 });
 230
 231                             return; // Don't double count `simd_shuffle` candidates
 232                         }
 233                     }
 234
 235                     if let Some(constant_args) = args_required_const(self.ccx.tcx, def_id) {
 236                         for index in constant_args {
 237                             self.candidates.push(Candidate::Argument { bb: location.block, index });
 238                         }
 239                     }
 240                 }
 241             }
 242             TerminatorKind::InlineAsm { ref operands, .. } => {
 243                 for (index, op) in operands.iter().enumerate() {
 244                     if let InlineAsmOperand::Const { .. } = op {
 245                         self.candidates.push(Candidate::InlineAsm { bb: location.block, index })
 246                     }
 247                 }
 248             }
 249             _ => {}
 250         }
 251     }
 252 }
 253
 254 pub fn collect_temps_and_candidates(
 255     ccx: &ConstCx<'mir, 'tcx>,
 256     rpo: &mut ReversePostorder<'_, 'tcx>,
 257 ) -> (IndexVec<Local, TempState>, Vec<Candidate>) {
 258     let mut collector = Collector {
 259         temps: IndexVec::from_elem(TempState::Undefined, &ccx.body.local_decls),
 260         candidates: vec![],
 261         ccx,
 262     };
 263     for (bb, data) in rpo {
 264         collector.visit_basic_block_data(bb, data);
 265     }
 266     (collector.temps, collector.candidates)
 267 }
 268
 269 /// Checks whether locals that appear in a promotion context (`Candidate`) are actually promotable.
 270 ///
 271 /// This wraps an `Item`, and has access to all fields of that `Item` via `Deref` coercion.
 272 struct Validator<'a, 'tcx> {
 273     ccx: &'a ConstCx<'a, 'tcx>,
 274     temps: &'a IndexVec<Local, TempState>,
 275
 276     /// Explicit promotion happens e.g. for constant arguments declared via
 277     /// `rustc_args_required_const`.
 278     /// Implicit promotion has almost the same rules, except that disallows `const fn`
 279     /// except for those marked `#[rustc_promotable]`. This is to avoid changing
 280     /// a legitimate run-time operation into a failing compile-time operation
 281     /// e.g. due to addresses being compared inside the function.
 282     explicit: bool,
 283 }
 284
 285 impl std::ops::Deref for Validator<'a, 'tcx> {
 286     type Target = ConstCx<'a, 'tcx>;
 287
 288     fn deref(&self) -> &Self::Target {
 289         &self.ccx
 290     }
 291 }
 292
 293 struct Unpromotable;
 294
 295 impl<'tcx> Validator<'_, 'tcx> {
 296     fn validate_candidate(&self, candidate: Candidate) -> Result<(), Unpromotable> {
 297         match candidate {
 298             Candidate::Ref(loc) => {
 299                 assert!(!self.explicit);
 300
 301                 let statement = &self.body[loc.block].statements[loc.statement_index];
 302                 match &statement.kind {
 303                     StatementKind::Assign(box (_, Rvalue::Ref(_, kind, place))) => {
 304                         // We can only promote interior borrows of promotable temps (non-temps
 305                         // don't get promoted anyway).
 306                         self.validate_local(place.local)?;
 307
 308                         // The reference operation itself must be promotable.
 309                         // (Needs to come after `validate_local` to avoid ICEs.)
 310                         self.validate_ref(*kind, place)?;
 311
 312                         // We do not check all the projections (they do not get promoted anyway),
 313                         // but we do stay away from promoting anything involving a dereference.
 314                         if place.projection.contains(&ProjectionElem::Deref) {
 315                             return Err(Unpromotable);
 316                         }
 317
 318                         // We cannot promote things that need dropping, since the promoted value
 319                         // would not get dropped.
 320                         if self.qualif_local::<qualifs::NeedsDrop>(place.local) {
 321                             return Err(Unpromotable);
 322                         }
 323
 324                         Ok(())
 325                     }
 326                     _ => bug!(),
 327                 }
 328             }
 329             Candidate::Argument { bb, index } => {
 330                 assert!(self.explicit);
 331
 332                 let terminator = self.body[bb].terminator();
 333                 match &terminator.kind {
 334                     TerminatorKind::Call { args, .. } => self.validate_operand(&args[index]),
 335                     _ => bug!(),
 336                 }
 337             }
 338             Candidate::InlineAsm { bb, index } => {
 339                 assert!(self.explicit);
 340
 341                 let terminator = self.body[bb].terminator();
 342                 match &terminator.kind {
 343                     TerminatorKind::InlineAsm { operands, .. } => match &operands[index] {
 344                         InlineAsmOperand::Const { value } => self.validate_operand(value),
 345                         _ => bug!(),
 346                     },
 347                     _ => bug!(),
 348                 }
 349             }
 350         }
 351     }
 352
 353     // FIXME(eddyb) maybe cache this?
 354     fn qualif_local<Q: qualifs::Qualif>(&self, local: Local) -> bool {
 355         if let TempState::Defined { location: loc, .. } = self.temps[local] {
 356             let num_stmts = self.body[loc.block].statements.len();
 357
 358             if loc.statement_index < num_stmts {
 359                 let statement = &self.body[loc.block].statements[loc.statement_index];
 360                 match &statement.kind {
 361                     StatementKind::Assign(box (_, rhs)) => qualifs::in_rvalue::<Q, _>(
 362                         &self.ccx,
 363                         &mut |l| self.qualif_local::<Q>(l),
 364                         rhs,
 365                     ),
 366                     _ => {
 367                         span_bug!(
 368                             statement.source_info.span,
 369                             "{:?} is not an assignment",
 370                             statement
 371                         );
 372                     }
 373                 }
 374             } else {
 375                 let terminator = self.body[loc.block].terminator();
 376                 match &terminator.kind {
 377                     TerminatorKind::Call { .. } => {
 378                         let return_ty = self.body.local_decls[local].ty;
 379                         Q::in_any_value_of_ty(&self.ccx, return_ty)
 380                     }
 381                     kind => {
 382                         span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
 383                     }
 384                 }
 385             }
 386         } else {
 387             let span = self.body.local_decls[local].source_info.span;
 388             span_bug!(span, "{:?} not promotable, qualif_local shouldn't have been called", local);
 389         }
 390     }
 391
 392     // FIXME(eddyb) maybe cache this?
 393     fn validate_local(&self, local: Local) -> Result<(), Unpromotable> {
 394         if let TempState::Defined { location: loc, .. } = self.temps[local] {
 395             let block = &self.body[loc.block];
 396             let num_stmts = block.statements.len();
 397
 398             if loc.statement_index < num_stmts {
 399                 let statement = &block.statements[loc.statement_index];
 400                 match &statement.kind {
 401                     StatementKind::Assign(box (_, rhs)) => self.validate_rvalue(rhs),
 402                     _ => {
 403                         span_bug!(
 404                             statement.source_info.span,
 405                             "{:?} is not an assignment",
 406                             statement
 407                         );
 408                     }
 409                 }
 410             } else {
 411                 let terminator = block.terminator();
 412                 match &terminator.kind {
 413                     TerminatorKind::Call { func, args, .. } => self.validate_call(func, args),
 414                     TerminatorKind::Yield { .. } => Err(Unpromotable),
 415                     kind => {
 416                         span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
 417                     }
 418                 }
 419             }
 420         } else {
 421             Err(Unpromotable)
 422         }
 423     }
 424
 425     fn validate_place(&self, place: PlaceRef<'tcx>) -> Result<(), Unpromotable> {
 426         match place.last_projection() {
 427             None => self.validate_local(place.local),
 428             Some((place_base, elem)) => {
 429                 // Validate topmost projection, then recurse.
 430                 match elem {
 431                     ProjectionElem::Deref => {
 432                         let mut promotable = false;
 433                         // We need to make sure this is a `Deref` of a local with no further projections.
 434                         // Discussion can be found at
 435                         // https://github.com/rust-lang/rust/pull/74945#discussion_r463063247
 436                         if let Some(local) = place_base.as_local() {
 437                             // This is a special treatment for cases like *&STATIC where STATIC is a
 438                             // global static variable.
 439                             // This pattern is generated only when global static variables are directly
 440                             // accessed and is qualified for promotion safely.
 441                             if let TempState::Defined { location, .. } = self.temps[local] {
 442                                 let def_stmt = self.body[location.block]
 443                                     .statements
 444                                     .get(location.statement_index);
 445                                 if let Some(Statement {
 446                                     kind:
 447                                         StatementKind::Assign(box (
 448                                             _,
 449                                             Rvalue::Use(Operand::Constant(c)),
 450                                         )),
 451                                     ..
 452                                 }) = def_stmt
 453                                 {
 454                                     if let Some(did) = c.check_static_ptr(self.tcx) {
 455                                         // Evaluating a promoted may not read statics except if it got
 456                                         // promoted from a static (this is a CTFE check). So we
 457                                         // can only promote static accesses inside statics.
 458                                         if let Some(hir::ConstContext::Static(..)) = self.const_kind
 459                                         {
 460                                             if !self.tcx.is_thread_local_static(did) {
 461                                                 promotable = true;
 462                                             }
 463                                         }
 464                                     }
 465                                 }
 466                             }
 467                         }
 468                         if !promotable {
 469                             return Err(Unpromotable);
 470                         }
 471                     }
 472                     ProjectionElem::Downcast(..) => {
 473                         return Err(Unpromotable);
 474                     }
 475
 476                     ProjectionElem::ConstantIndex { .. } | ProjectionElem::Subslice { .. } => {}
 477
 478                     ProjectionElem::Index(local) => {
 479                         if !self.explicit {
 480                             let mut promotable = false;
 481                             // Only accept if we can predict the index and are indexing an array.
 482                             let val = if let TempState::Defined { location: loc, .. } =
 483                                 self.temps[local]
 484                             {
 485                                 let block = &self.body[loc.block];
 486                                 if loc.statement_index < block.statements.len() {
 487                                     let statement = &block.statements[loc.statement_index];
 488                                     match &statement.kind {
 489                                         StatementKind::Assign(box (
 490                                             _,
 491                                             Rvalue::Use(Operand::Constant(c)),
 492                                         )) => c.literal.try_eval_usize(self.tcx, self.param_env),
 493                                         _ => None,
 494                                     }
 495                                 } else {
 496                                     None
 497                                 }
 498                             } else {
 499                                 None
 500                             };
 501                             if let Some(idx) = val {
 502                                 // Determine the type of the thing we are indexing.
 503                                 let ty = place_base.ty(self.body, self.tcx).ty;
 504                                 match ty.kind() {
 505                                     ty::Array(_, len) => {
 506                                         // It's an array; determine its length.
 507                                         if let Some(len) =
 508                                             len.try_eval_usize(self.tcx, self.param_env)
 509                                         {
 510                                             // If the index is in-bounds, go ahead.
 511                                             if idx < len {
 512                                                 promotable = true;
 513                                             }
 514                                         }
 515                                     }
 516                                     _ => {}
 517                                 }
 518                             }
 519                             if !promotable {
 520                                 return Err(Unpromotable);
 521                             }
 522                         }
 523                         self.validate_local(local)?;
 524                     }
 525
 526                     ProjectionElem::Field(..) => {
 527                         let base_ty = place_base.ty(self.body, self.tcx).ty;
 528                         if let Some(def) = base_ty.ty_adt_def() {
 529                             // No promotion of union field accesses.
 530                             if def.is_union() {
 531                                 return Err(Unpromotable);
 532                             }
 533                         }
 534                     }
 535                 }
 536
 537                 self.validate_place(place_base)
 538             }
 539         }
 540     }
 541
 542     fn validate_operand(&self, operand: &Operand<'tcx>) -> Result<(), Unpromotable> {
 543         match operand {
 544             Operand::Copy(place) | Operand::Move(place) => self.validate_place(place.as_ref()),
 545
 546             // The qualifs for a constant (e.g. `HasMutInterior`) are checked in
 547             // `validate_rvalue` upon access.
 548             Operand::Constant(c) => {
 549                 if let Some(def_id) = c.check_static_ptr(self.tcx) {
 550                     // Only allow statics (not consts) to refer to other statics.
 551                     // FIXME(eddyb) does this matter at all for promotion?
 552                     // FIXME(RalfJung) it makes little sense to not promote this in `fn`/`const fn`,
 553                     // and in `const` this cannot occur anyway. The only concern is that we might
 554                     // promote even `let x = &STATIC` which would be useless, but this applies to
 555                     // promotion inside statics as well.
 556                     let is_static = matches!(self.const_kind, Some(hir::ConstContext::Static(_)));
 557                     if !is_static {
 558                         return Err(Unpromotable);
 559                     }
 560
 561                     let is_thread_local = self.tcx.is_thread_local_static(def_id);
 562                     if is_thread_local {
 563                         return Err(Unpromotable);
 564                     }
 565                 }
 566
 567                 Ok(())
 568             }
 569         }
 570     }
 571
 572     fn validate_ref(&self, kind: BorrowKind, place: &Place<'tcx>) -> Result<(), Unpromotable> {
 573         match kind {
 574             // Reject these borrow types just to be safe.
 575             // FIXME(RalfJung): could we allow them? Should we? No point in it until we have a usecase.
 576             BorrowKind::Shallow | BorrowKind::Unique => return Err(Unpromotable),
 577
 578             BorrowKind::Shared => {
 579                 let has_mut_interior = self.qualif_local::<qualifs::HasMutInterior>(place.local);
 580                 if has_mut_interior {
 581                     return Err(Unpromotable);
 582                 }
 583             }
 584
 585             BorrowKind::Mut { .. } => {
 586                 let ty = place.ty(self.body, self.tcx).ty;
 587
 588                 // In theory, any zero-sized value could be borrowed
 589                 // mutably without consequences. However, only &mut []
 590                 // is allowed right now.
 591                 if let ty::Array(_, len) = ty.kind() {
 592                     match len.try_eval_usize(self.tcx, self.param_env) {
 593                         Some(0) => {}
 594                         _ => return Err(Unpromotable),
 595                     }
 596                 } else {
 597                     return Err(Unpromotable);
 598                 }
 599             }
 600         }
 601
 602         Ok(())
 603     }
 604
 605     fn validate_rvalue(&self, rvalue: &Rvalue<'tcx>) -> Result<(), Unpromotable> {
 606         match rvalue {
 607             Rvalue::Use(operand) | Rvalue::Repeat(operand, _) => {
 608                 self.validate_operand(operand)?;
 609             }
 610
 611             Rvalue::Discriminant(place) | Rvalue::Len(place) => {
 612                 self.validate_place(place.as_ref())?
 613             }
 614
 615             Rvalue::ThreadLocalRef(_) => return Err(Unpromotable),
 616
 617             Rvalue::Cast(kind, operand, cast_ty) => {
 618                 if matches!(kind, CastKind::Misc) {
 619                     let operand_ty = operand.ty(self.body, self.tcx);
 620                     let cast_in = CastTy::from_ty(operand_ty).expect("bad input type for cast");
 621                     let cast_out = CastTy::from_ty(cast_ty).expect("bad output type for cast");
 622                     if let (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Int(_)) = (cast_in, cast_out) {
 623                         // ptr-to-int casts are not possible in consts and thus not promotable
 624                         return Err(Unpromotable);
 625                     }
 626                     // int-to-ptr casts are fine, they just use the integer value at pointer type.
 627                 }
 628
 629                 self.validate_operand(operand)?;
 630             }
 631
 632             Rvalue::NullaryOp(op, _) => match op {
 633                 NullOp::Box => return Err(Unpromotable),
 634                 NullOp::SizeOf => {}
 635             },
 636
 637             Rvalue::UnaryOp(op, operand) => {
 638                 match op {
 639                     // These operations can never fail.
 640                     UnOp::Neg | UnOp::Not => {}
 641                 }
 642
 643                 self.validate_operand(operand)?;
 644             }
 645
 646             Rvalue::BinaryOp(op, lhs, rhs) | Rvalue::CheckedBinaryOp(op, lhs, rhs) => {
 647                 let op = *op;
 648                 let lhs_ty = lhs.ty(self.body, self.tcx);
 649
 650                 if let ty::RawPtr(_) | ty::FnPtr(..) = lhs_ty.kind() {
 651                     // Raw and fn pointer operations are not allowed inside consts and thus not promotable.
 652                     assert!(matches!(
 653                         op,
 654                         BinOp::Eq
 655                             | BinOp::Ne
 656                             | BinOp::Le
 657                             | BinOp::Lt
 658                             | BinOp::Ge
 659                             | BinOp::Gt
 660                             | BinOp::Offset
 661                     ));
 662                     return Err(Unpromotable);
 663                 }
 664
 665                 match op {
 666                     BinOp::Div | BinOp::Rem => {
 667                         if !self.explicit && lhs_ty.is_integral() {
 668                             // Integer division: the RHS must be a non-zero const.
 669                             let const_val = match rhs {
 670                                 Operand::Constant(c) => {
 671                                     c.literal.try_eval_bits(self.tcx, self.param_env, lhs_ty)
 672                                 }
 673                                 _ => None,
 674                             };
 675                             match const_val {
 676                                 Some(x) if x != 0 => {}        // okay
 677                                 _ => return Err(Unpromotable), // value not known or 0 -- not okay
 678                             }
 679                         }
 680                     }
 681                     // The remaining operations can never fail.
 682                     BinOp::Eq
 683                     | BinOp::Ne
 684                     | BinOp::Le
 685                     | BinOp::Lt
 686                     | BinOp::Ge
 687                     | BinOp::Gt
 688                     | BinOp::Offset
 689                     | BinOp::Add
 690                     | BinOp::Sub
 691                     | BinOp::Mul
 692                     | BinOp::BitXor
 693                     | BinOp::BitAnd
 694                     | BinOp::BitOr
 695                     | BinOp::Shl
 696                     | BinOp::Shr => {}
 697                 }
 698
 699                 self.validate_operand(lhs)?;
 700                 self.validate_operand(rhs)?;
 701             }
 702
 703             Rvalue::AddressOf(_, place) => {
 704                 // We accept `&raw *`, i.e., raw reborrows -- creating a raw pointer is
 705                 // no problem, only using it is.
 706                 if let Some((place_base, ProjectionElem::Deref)) = place.as_ref().last_projection()
 707                 {
 708                     let base_ty = place_base.ty(self.body, self.tcx).ty;
 709                     if let ty::Ref(..) = base_ty.kind() {
 710                         return self.validate_place(place_base);
 711                     }
 712                 }
 713                 return Err(Unpromotable);
 714             }
 715
 716             Rvalue::Ref(_, kind, place) => {
 717                 // Special-case reborrows to be more like a copy of the reference.
 718                 let mut place_simplified = place.as_ref();
 719                 if let Some((place_base, ProjectionElem::Deref)) =
 720                     place_simplified.last_projection()
 721                 {
 722                     let base_ty = place_base.ty(self.body, self.tcx).ty;
 723                     if let ty::Ref(..) = base_ty.kind() {
 724                         place_simplified = place_base;
 725                     }
 726                 }
 727
 728                 self.validate_place(place_simplified)?;
 729
 730                 // Check that the reference is fine (using the original place!).
 731                 // (Needs to come after `validate_place` to avoid ICEs.)
 732                 self.validate_ref(*kind, place)?;
 733             }
 734
 735             Rvalue::Aggregate(_, operands) => {
 736                 for o in operands {
 737                     self.validate_operand(o)?;
 738                 }
 739             }
 740         }
 741
 742         Ok(())
 743     }
 744
 745     fn validate_call(
 746         &self,
 747         callee: &Operand<'tcx>,
 748         args: &[Operand<'tcx>],
 749     ) -> Result<(), Unpromotable> {
 750         let fn_ty = callee.ty(self.body, self.tcx);
 751
 752         // When doing explicit promotion and inside const/static items, we promote all (eligible) function calls.
 753         // Everywhere else, we require `#[rustc_promotable]` on the callee.
 754         let promote_all_const_fn = self.explicit
 755             || matches!(
 756                 self.const_kind,
 757                 Some(hir::ConstContext::Static(_) | hir::ConstContext::Const)
 758             );
 759         if !promote_all_const_fn {
 760             if let ty::FnDef(def_id, _) = *fn_ty.kind() {
 761                 // Never promote runtime `const fn` calls of
 762                 // functions without `#[rustc_promotable]`.
 763                 if !self.tcx.is_promotable_const_fn(def_id) {
 764                     return Err(Unpromotable);
 765                 }
 766             }
 767         }
 768
 769         let is_const_fn = match *fn_ty.kind() {
 770             ty::FnDef(def_id, _) => {
 771                 is_const_fn(self.tcx, def_id)
 772                     || is_unstable_const_fn(self.tcx, def_id).is_some()
 773                     || is_lang_panic_fn(self.tcx, def_id)
 774             }
 775             _ => false,
 776         };
 777         if !is_const_fn {
 778             return Err(Unpromotable);
 779         }
 780
 781         self.validate_operand(callee)?;
 782         for arg in args {
 783             self.validate_operand(arg)?;
 784         }
 785
 786         Ok(())
 787     }
 788 }
 789
 790 // FIXME(eddyb) remove the differences for promotability in `static`, `const`, `const fn`.
 791 pub fn validate_candidates(
 792     ccx: &ConstCx<'_, '_>,
 793     temps: &IndexVec<Local, TempState>,
 794     candidates: &[Candidate],
 795 ) -> Vec<Candidate> {
 796     let mut validator = Validator { ccx, temps, explicit: false };
 797
 798     candidates
 799         .iter()
 800         .copied()
 801         .filter(|&candidate| {
 802             validator.explicit = candidate.forces_explicit_promotion();
 803
 804             // FIXME(eddyb) also emit the errors for shuffle indices
 805             // and `#[rustc_args_required_const]` arguments here.
 806
 807             let is_promotable = validator.validate_candidate(candidate).is_ok();
 808
 809             // If we use explicit validation, we carry the risk of turning a legitimate run-time
 810             // operation into a failing compile-time operation. Make sure that does not happen
 811             // by asserting that there is no possible run-time behavior here in case promotion
 812             // fails.
 813             if validator.explicit && !is_promotable {
 814                 ccx.tcx.sess.delay_span_bug(
 815                     ccx.body.span,
 816                     "Explicit promotion requested, but failed to promote",
 817                 );
 818             }
 819
 820             match candidate {
 821                 Candidate::Argument { bb, index } | Candidate::InlineAsm { bb, index }
 822                     if !is_promotable =>
 823                 {
 824                     let span = ccx.body[bb].terminator().source_info.span;
 825                     let msg = format!("argument {} is required to be a constant", index + 1);
 826                     ccx.tcx.sess.span_err(span, &msg);
 827                 }
 828                 _ => (),
 829             }
 830
 831             is_promotable
 832         })
 833         .collect()
 834 }
 835
 836 struct Promoter<'a, 'tcx> {
 837     tcx: TyCtxt<'tcx>,
 838     source: &'a mut Body<'tcx>,
 839     promoted: Body<'tcx>,
 840     temps: &'a mut IndexVec<Local, TempState>,
 841     extra_statements: &'a mut Vec<(Location, Statement<'tcx>)>,
 842
 843     /// If true, all nested temps are also kept in the
 844     /// source MIR, not moved to the promoted MIR.
 845     keep_original: bool,
 846 }
 847
 848 impl<'a, 'tcx> Promoter<'a, 'tcx> {
 849     fn new_block(&mut self) -> BasicBlock {
 850         let span = self.promoted.span;
 851         self.promoted.basic_blocks_mut().push(BasicBlockData {
 852             statements: vec![],
 853             terminator: Some(Terminator {
 854                 source_info: SourceInfo::outermost(span),
 855                 kind: TerminatorKind::Return,
 856             }),
 857             is_cleanup: false,
 858         })
 859     }
 860
 861     fn assign(&mut self, dest: Local, rvalue: Rvalue<'tcx>, span: Span) {
 862         let last = self.promoted.basic_blocks().last().unwrap();
 863         let data = &mut self.promoted[last];
 864         data.statements.push(Statement {
 865             source_info: SourceInfo::outermost(span),
 866             kind: StatementKind::Assign(box (Place::from(dest), rvalue)),
 867         });
 868     }
 869
 870     fn is_temp_kind(&self, local: Local) -> bool {
 871         self.source.local_kind(local) == LocalKind::Temp
 872     }
 873
 874     /// Copies the initialization of this temp to the
 875     /// promoted MIR, recursing through temps.
 876     fn promote_temp(&mut self, temp: Local) -> Local {
 877         let old_keep_original = self.keep_original;
 878         let loc = match self.temps[temp] {
 879             TempState::Defined { location, uses } if uses > 0 => {
 880                 if uses > 1 {
 881                     self.keep_original = true;
 882                 }
 883                 location
 884             }
 885             state => {
 886                 span_bug!(self.promoted.span, "{:?} not promotable: {:?}", temp, state);
 887             }
 888         };
 889         if !self.keep_original {
 890             self.temps[temp] = TempState::PromotedOut;
 891         }
 892
 893         let num_stmts = self.source[loc.block].statements.len();
 894         let new_temp = self.promoted.local_decls.push(LocalDecl::new(
 895             self.source.local_decls[temp].ty,
 896             self.source.local_decls[temp].source_info.span,
 897         ));
 898
 899         debug!("promote({:?} @ {:?}/{:?}, {:?})", temp, loc, num_stmts, self.keep_original);
 900
 901         // First, take the Rvalue or Call out of the source MIR,
 902         // or duplicate it, depending on keep_original.
 903         if loc.statement_index < num_stmts {
 904             let (mut rvalue, source_info) = {
 905                 let statement = &mut self.source[loc.block].statements[loc.statement_index];
 906                 let rhs = match statement.kind {
 907                     StatementKind::Assign(box (_, ref mut rhs)) => rhs,
 908                     _ => {
 909                         span_bug!(
 910                             statement.source_info.span,
 911                             "{:?} is not an assignment",
 912                             statement
 913                         );
 914                     }
 915                 };
 916
 917                 (
 918                     if self.keep_original {
 919                         rhs.clone()
 920                     } else {
 921                         let unit = Rvalue::Use(Operand::Constant(box Constant {
 922                             span: statement.source_info.span,
 923                             user_ty: None,
 924                             literal: ty::Const::zero_sized(self.tcx, self.tcx.types.unit),
 925                         }));
 926                         mem::replace(rhs, unit)
 927                     },
 928                     statement.source_info,
 929                 )
 930             };
 931
 932             self.visit_rvalue(&mut rvalue, loc);
 933             self.assign(new_temp, rvalue, source_info.span);
 934         } else {
 935             let terminator = if self.keep_original {
 936                 self.source[loc.block].terminator().clone()
 937             } else {
 938                 let terminator = self.source[loc.block].terminator_mut();
 939                 let target = match terminator.kind {
 940                     TerminatorKind::Call { destination: Some((_, target)), .. } => target,
 941                     ref kind => {
 942                         span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
 943                     }
 944                 };
 945                 Terminator {
 946                     source_info: terminator.source_info,
 947                     kind: mem::replace(&mut terminator.kind, TerminatorKind::Goto { target }),
 948                 }
 949             };
 950
 951             match terminator.kind {
 952                 TerminatorKind::Call { mut func, mut args, from_hir_call, fn_span, .. } => {
 953                     self.visit_operand(&mut func, loc);
 954                     for arg in &mut args {
 955                         self.visit_operand(arg, loc);
 956                     }
 957
 958                     let last = self.promoted.basic_blocks().last().unwrap();
 959                     let new_target = self.new_block();
 960
 961                     *self.promoted[last].terminator_mut() = Terminator {
 962                         kind: TerminatorKind::Call {
 963                             func,
 964                             args,
 965                             cleanup: None,
 966                             destination: Some((Place::from(new_temp), new_target)),
 967                             from_hir_call,
 968                             fn_span,
 969                         },
 970                         source_info: SourceInfo::outermost(terminator.source_info.span),
 971                         ..terminator
 972                     };
 973                 }
 974                 ref kind => {
 975                     span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
 976                 }
 977             };
 978         };
 979
 980         self.keep_original = old_keep_original;
 981         new_temp
 982     }
 983
 984     fn promote_candidate(
 985         mut self,
 986         candidate: Candidate,
 987         next_promoted_id: usize,
 988     ) -> Option<Body<'tcx>> {
 989         let def = self.source.source.with_opt_param();
 990         let mut rvalue = {
 991             let promoted = &mut self.promoted;
 992             let promoted_id = Promoted::new(next_promoted_id);
 993             let tcx = self.tcx;
 994             let mut promoted_operand = |ty, span| {
 995                 promoted.span = span;
 996                 promoted.local_decls[RETURN_PLACE] = LocalDecl::new(ty, span);
 997
 998                 Operand::Constant(Box::new(Constant {
 999                     span,
1000                     user_ty: None,
1001                     literal: tcx.mk_const(ty::Const {
1002                         ty,
1003                         val: ty::ConstKind::Unevaluated(
1004                             def,
1005                             InternalSubsts::for_item(tcx, def.did, |param, _| {
1006                                 if let ty::GenericParamDefKind::Lifetime = param.kind {
1007                                     tcx.lifetimes.re_erased.into()
1008                                 } else {
1009                                     tcx.mk_param_from_def(param)
1010                                 }
1011                             }),
1012                             Some(promoted_id),
1013                         ),
1014                     }),
1015                 }))
1016             };
1017             let (blocks, local_decls) = self.source.basic_blocks_and_local_decls_mut();
1018             match candidate {
1019                 Candidate::Ref(loc) => {
1020                     let statement = &mut blocks[loc.block].statements[loc.statement_index];
1021                     match statement.kind {
1022                         StatementKind::Assign(box (
1023                             _,
1024                             Rvalue::Ref(ref mut region, borrow_kind, ref mut place),
1025                         )) => {
1026                             // Use the underlying local for this (necessarily interior) borrow.
1027                             let ty = local_decls.local_decls()[place.local].ty;
1028                             let span = statement.source_info.span;
1029
1030                             let ref_ty = tcx.mk_ref(
1031                                 tcx.lifetimes.re_erased,
1032                                 ty::TypeAndMut { ty, mutbl: borrow_kind.to_mutbl_lossy() },
1033                             );
1034
1035                             *region = tcx.lifetimes.re_erased;
1036
1037                             let mut projection = vec![PlaceElem::Deref];
1038                             projection.extend(place.projection);
1039                             place.projection = tcx.intern_place_elems(&projection);
1040
1041                             // Create a temp to hold the promoted reference.
1042                             // This is because `*r` requires `r` to be a local,
1043                             // otherwise we would use the `promoted` directly.
1044                             let mut promoted_ref = LocalDecl::new(ref_ty, span);
1045                             promoted_ref.source_info = statement.source_info;
1046                             let promoted_ref = local_decls.push(promoted_ref);
1047                             assert_eq!(self.temps.push(TempState::Unpromotable), promoted_ref);
1048
1049                             let promoted_ref_statement = Statement {
1050                                 source_info: statement.source_info,
1051                                 kind: StatementKind::Assign(Box::new((
1052                                     Place::from(promoted_ref),
1053                                     Rvalue::Use(promoted_operand(ref_ty, span)),
1054                                 ))),
1055                             };
1056                             self.extra_statements.push((loc, promoted_ref_statement));
1057
1058                             Rvalue::Ref(
1059                                 tcx.lifetimes.re_erased,
1060                                 borrow_kind,
1061                                 Place {
1062                                     local: mem::replace(&mut place.local, promoted_ref),
1063                                     projection: List::empty(),
1064                                 },
1065                             )
1066                         }
1067                         _ => bug!(),
1068                     }
1069                 }
1070                 Candidate::Argument { bb, index } => {
1071                     let terminator = blocks[bb].terminator_mut();
1072                     match terminator.kind {
1073                         TerminatorKind::Call { ref mut args, .. } => {
1074                             let ty = args[index].ty(local_decls, self.tcx);
1075                             let span = terminator.source_info.span;
1076
1077                             Rvalue::Use(mem::replace(&mut args[index], promoted_operand(ty, span)))
1078                         }
1079                         // We expected a `TerminatorKind::Call` for which we'd like to promote an
1080                         // argument. `qualify_consts` saw a `TerminatorKind::Call` here, but
1081                         // we are seeing a `Goto`. That means that the `promote_temps` method
1082                         // already promoted this call away entirely. This case occurs when calling
1083                         // a function requiring a constant argument and as that constant value
1084                         // providing a value whose computation contains another call to a function
1085                         // requiring a constant argument.
1086                         TerminatorKind::Goto { .. } => return None,
1087                         _ => bug!(),
1088                     }
1089                 }
1090                 Candidate::InlineAsm { bb, index } => {
1091                     let terminator = blocks[bb].terminator_mut();
1092                     match terminator.kind {
1093                         TerminatorKind::InlineAsm { ref mut operands, .. } => {
1094                             match &mut operands[index] {
1095                                 InlineAsmOperand::Const { ref mut value } => {
1096                                     let ty = value.ty(local_decls, self.tcx);
1097                                     let span = terminator.source_info.span;
1098
1099                                     Rvalue::Use(mem::replace(value, promoted_operand(ty, span)))
1100                                 }
1101                                 _ => bug!(),
1102                             }
1103                         }
1104
1105                         _ => bug!(),
1106                     }
1107                 }
1108             }
1109         };
1110
1111         assert_eq!(self.new_block(), START_BLOCK);
1112         self.visit_rvalue(
1113             &mut rvalue,
1114             Location { block: BasicBlock::new(0), statement_index: usize::MAX },
1115         );
1116
1117         let span = self.promoted.span;
1118         self.assign(RETURN_PLACE, rvalue, span);
1119         Some(self.promoted)
1120     }
1121 }
1122
1123 /// Replaces all temporaries with their promoted counterparts.
1124 impl<'a, 'tcx> MutVisitor<'tcx> for Promoter<'a, 'tcx> {
1125     fn tcx(&self) -> TyCtxt<'tcx> {
1126         self.tcx
1127     }
1128
1129     fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _: Location) {
1130         if self.is_temp_kind(*local) {
1131             *local = self.promote_temp(*local);
1132         }
1133     }
1134 }
1135
1136 pub fn promote_candidates<'tcx>(
1137     body: &mut Body<'tcx>,
1138     tcx: TyCtxt<'tcx>,
1139     mut temps: IndexVec<Local, TempState>,
1140     candidates: Vec<Candidate>,
1141 ) -> IndexVec<Promoted, Body<'tcx>> {
1142     // Visit candidates in reverse, in case they're nested.
1143     debug!("promote_candidates({:?})", candidates);
1144
1145     let mut promotions = IndexVec::new();
1146
1147     let mut extra_statements = vec![];
1148     for candidate in candidates.into_iter().rev() {
1149         match candidate {
1150             Candidate::Ref(Location { block, statement_index }) => {
1151                 if let StatementKind::Assign(box (place, _)) =
1152                     &body[block].statements[statement_index].kind
1153                 {
1154                     if let Some(local) = place.as_local() {
1155                         if temps[local] == TempState::PromotedOut {
1156                             // Already promoted.
1157                             continue;
1158                         }
1159                     }
1160                 }
1161             }
1162             Candidate::Argument { .. } | Candidate::InlineAsm { .. } => {}
1163         }
1164
1165         // Declare return place local so that `mir::Body::new` doesn't complain.
1166         let initial_locals = iter::once(LocalDecl::new(tcx.types.never, body.span)).collect();
1167
1168         let mut scope = body.source_scopes[candidate.source_info(body).scope].clone();
1169         scope.parent_scope = None;
1170
1171         let promoted = Body::new(
1172             body.source, // `promoted` gets filled in below
1173             IndexVec::new(),
1174             IndexVec::from_elem_n(scope, 1),
1175             initial_locals,
1176             IndexVec::new(),
1177             0,
1178             vec![],
1179             body.span,
1180             body.generator_kind,
1181         );
1182
1183         let promoter = Promoter {
1184             promoted,
1185             tcx,
1186             source: body,
1187             temps: &mut temps,
1188             extra_statements: &mut extra_statements,
1189             keep_original: false,
1190         };
1191
1192         //FIXME(oli-obk): having a `maybe_push()` method on `IndexVec` might be nice
1193         if let Some(mut promoted) = promoter.promote_candidate(candidate, promotions.len()) {
1194             promoted.source.promoted = Some(promotions.next_index());
1195             promotions.push(promoted);
1196         }
1197     }
1198
1199     // Insert each of `extra_statements` before its indicated location, which
1200     // has to be done in reverse location order, to not invalidate the rest.
1201     extra_statements.sort_by_key(|&(loc, _)| cmp::Reverse(loc));
1202     for (loc, statement) in extra_statements {
1203         body[loc.block].statements.insert(loc.statement_index, statement);
1204     }
1205
1206     // Eliminate assignments to, and drops of promoted temps.
1207     let promoted = |index: Local| temps[index] == TempState::PromotedOut;
1208     for block in body.basic_blocks_mut() {
1209         block.statements.retain(|statement| match &statement.kind {
1210             StatementKind::Assign(box (place, _)) => {
1211                 if let Some(index) = place.as_local() {
1212                     !promoted(index)
1213                 } else {
1214                     true
1215                 }
1216             }
1217             StatementKind::StorageLive(index) | StatementKind::StorageDead(index) => {
1218                 !promoted(*index)
1219             }
1220             _ => true,
1221         });
1222         let terminator = block.terminator_mut();
1223         if let TerminatorKind::Drop { place, target, .. } = &terminator.kind {
1224             if let Some(index) = place.as_local() {
1225                 if promoted(index) {
1226                     terminator.kind = TerminatorKind::Goto { target: *target };
1227                 }
1228             }
1229         }
1230     }
1231
1232     promotions
1233 }
1234
1235 /// This function returns `true` if the function being called in the array
1236 /// repeat expression is a `const` function.
1237 crate fn is_const_fn_in_array_repeat_expression<'tcx>(
1238     ccx: &ConstCx<'_, 'tcx>,
1239     place: &Place<'tcx>,
1240     body: &Body<'tcx>,
1241 ) -> bool {
1242     match place.as_local() {
1243         // rule out cases such as: `let my_var = some_fn(); [my_var; N]`
1244         Some(local) if body.local_decls[local].is_user_variable() => return false,
1245         None => return false,
1246         _ => {}
1247     }
1248
1249     for block in body.basic_blocks() {
1250         if let Some(Terminator { kind: TerminatorKind::Call { func, destination, .. }, .. }) =
1251             &block.terminator
1252         {
1253             if let Operand::Constant(box Constant { literal: ty::Const { ty, .. }, .. }) = func {
1254                 if let ty::FnDef(def_id, _) = *ty.kind() {
1255                     if let Some((destination_place, _)) = destination {
1256                         if destination_place == place {
1257                             if is_const_fn(ccx.tcx, def_id) {
1258                                 return true;
1259                             }
1260                         }
1261                     }
1262                 }
1263             }
1264         }
1265     }
1266
1267     false
1268 }