From: Markus Westerlind Date: Mon, 24 Feb 2020 09:40:36 +0000 (+0100) Subject: perf: Reduce snapshot/rollback overhead X-Git-Url: https://git.lizzy.rs/?a=commitdiff_plain;h=1506b1fc6a42dda3e56789f43587448cc5bf4a36;p=rust.git perf: Reduce snapshot/rollback overhead By merging the undo_log of all structures part of the snapshot the cost of creating a snapshot becomes much cheaper. Since snapshots with no or few changes are so frequent this ends up mattering more than the slight overhead of dispatching on the variants that map to each field. --- diff --git a/Cargo.lock b/Cargo.lock index 74578084a72..6f442e8d2b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -987,8 +987,7 @@ dependencies = [ [[package]] name = "ena" version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8944dc8fa28ce4a38f778bd46bf7d923fe73eed5a439398507246c8e017e6f36" +source = "git+https://github.com/Marwes/ena?branch=detach_undo_log#9b977ea7f209a35f46d65d33cdd74b8f4931fb8a" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index 7b5e0fa1c28..9b143dcc8d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,5 +65,7 @@ rustc-std-workspace-core = { path = 'src/tools/rustc-std-workspace-core' } rustc-std-workspace-alloc = { path = 'src/tools/rustc-std-workspace-alloc' } rustc-std-workspace-std = { path = 'src/tools/rustc-std-workspace-std' } +ena = { version = "0.13.1", git = "https://github.com/Marwes/ena", branch = "detach_undo_log" } + [patch."https://github.com/rust-lang/rust-clippy"] clippy_lints = { path = "src/tools/clippy/clippy_lints" } diff --git a/src/librustc_data_structures/lib.rs b/src/librustc_data_structures/lib.rs index bc2da535fd3..a7bee8a067c 100644 --- a/src/librustc_data_structures/lib.rs +++ b/src/librustc_data_structures/lib.rs @@ -84,6 +84,7 @@ macro_rules! unlikely { pub mod thin_vec; pub mod tiny_list; pub mod transitive_relation; +pub use ena::undo_log; pub use ena::unify; mod atomic_ref; pub mod fingerprint; diff --git a/src/librustc_infer/infer/combine.rs b/src/librustc_infer/infer/combine.rs index b03044b72da..d4af4704996 100644 --- a/src/librustc_infer/infer/combine.rs +++ b/src/librustc_infer/infer/combine.rs @@ -76,7 +76,7 @@ pub fn super_combine_tys( (&ty::Infer(ty::IntVar(a_id)), &ty::Infer(ty::IntVar(b_id))) => { self.inner .borrow_mut() - .int_unification_table + .int_unification_table() .unify_var_var(a_id, b_id) .map_err(|e| int_unification_error(a_is_expected, e))?; Ok(a) @@ -98,7 +98,7 @@ pub fn super_combine_tys( (&ty::Infer(ty::FloatVar(a_id)), &ty::Infer(ty::FloatVar(b_id))) => { self.inner .borrow_mut() - .float_unification_table + .float_unification_table() .unify_var_var(a_id, b_id) .map_err(|e| float_unification_error(relation.a_is_expected(), e))?; Ok(a) @@ -133,8 +133,8 @@ pub fn super_combine_consts( return Ok(a); } - let a = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table, a); - let b = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table, b); + let a = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table(), a); + let b = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table(), b); let a_is_expected = relation.a_is_expected(); @@ -145,7 +145,7 @@ pub fn super_combine_consts( ) => { self.inner .borrow_mut() - .const_unification_table + .const_unification_table() .unify_var_var(a_vid, b_vid) .map_err(|e| const_unification_error(a_is_expected, e))?; return Ok(a); @@ -179,7 +179,7 @@ pub fn unify_const_variable( ) -> RelateResult<'tcx, &'tcx ty::Const<'tcx>> { self.inner .borrow_mut() - .const_unification_table + .const_unification_table() .unify_var_value( vid, ConstVarValue { @@ -202,7 +202,7 @@ fn unify_integral_variable( ) -> RelateResult<'tcx, Ty<'tcx>> { self.inner .borrow_mut() - .int_unification_table + .int_unification_table() .unify_var_value(vid, Some(val)) .map_err(|e| int_unification_error(vid_is_expected, e))?; match val { @@ -219,7 +219,7 @@ fn unify_float_variable( ) -> RelateResult<'tcx, Ty<'tcx>> { self.inner .borrow_mut() - .float_unification_table + .float_unification_table() .unify_var_value(vid, Some(ty::FloatVarValue(val))) .map_err(|e| float_unification_error(vid_is_expected, e))?; Ok(self.tcx.mk_mach_float(val)) @@ -266,7 +266,7 @@ pub fn instantiate( use self::RelationDir::*; // Get the actual variable that b_vid has been inferred to - debug_assert!(self.infcx.inner.borrow_mut().type_variables.probe(b_vid).is_unknown()); + debug_assert!(self.infcx.inner.borrow_mut().type_variables().probe(b_vid).is_unknown()); debug!("instantiate(a_ty={:?} dir={:?} b_vid={:?})", a_ty, dir, b_vid); @@ -286,7 +286,7 @@ pub fn instantiate( "instantiate(a_ty={:?}, dir={:?}, b_vid={:?}, generalized b_ty={:?})", a_ty, dir, b_vid, b_ty ); - self.infcx.inner.borrow_mut().type_variables.instantiate(b_vid, b_ty); + self.infcx.inner.borrow_mut().type_variables().instantiate(b_vid, b_ty); if needs_wf { self.obligations.push(Obligation::new( @@ -344,7 +344,7 @@ fn generalize( debug!("generalize: ambient_variance = {:?}", ambient_variance); - let for_universe = match self.infcx.inner.borrow_mut().type_variables.probe(for_vid) { + let for_universe = match self.infcx.inner.borrow_mut().type_variables().probe(for_vid) { v @ TypeVariableValue::Known { .. } => { panic!("instantiating {:?} which has a known value {:?}", for_vid, v,) } @@ -356,7 +356,7 @@ fn generalize( let mut generalize = Generalizer { infcx: self.infcx, span: self.trace.cause.span, - for_vid_sub_root: self.infcx.inner.borrow_mut().type_variables.sub_root_var(for_vid), + for_vid_sub_root: self.infcx.inner.borrow_mut().type_variables().sub_root_var(for_vid), for_universe, ambient_variance, needs_wf: false, @@ -508,14 +508,14 @@ fn tys(&mut self, t: Ty<'tcx>, t2: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> { // us from creating infinitely sized types. match t.kind { ty::Infer(ty::TyVar(vid)) => { - let vid = self.infcx.inner.borrow_mut().type_variables.root_var(vid); - let sub_vid = self.infcx.inner.borrow_mut().type_variables.sub_root_var(vid); + let vid = self.infcx.inner.borrow_mut().type_variables().root_var(vid); + let sub_vid = self.infcx.inner.borrow_mut().type_variables().sub_root_var(vid); if sub_vid == self.for_vid_sub_root { // If sub-roots are equal, then `for_vid` and // `vid` are related via subtyping. Err(TypeError::CyclicTy(self.root_ty)) } else { - let probe = self.infcx.inner.borrow_mut().type_variables.probe(vid); + let probe = self.infcx.inner.borrow_mut().type_variables().probe(vid); match probe { TypeVariableValue::Known { value: u } => { debug!("generalize: known value {:?}", u); @@ -542,12 +542,13 @@ fn tys(&mut self, t: Ty<'tcx>, t2: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> { } let origin = - *self.infcx.inner.borrow_mut().type_variables.var_origin(vid); - let new_var_id = self.infcx.inner.borrow_mut().type_variables.new_var( - self.for_universe, - false, - origin, - ); + *self.infcx.inner.borrow_mut().type_variables().var_origin(vid); + let new_var_id = self + .infcx + .inner + .borrow_mut() + .type_variables() + .new_var(self.for_universe, false, origin); let u = self.tcx().mk_ty_var(new_var_id); debug!("generalize: replacing original vid={:?} with new={:?}", vid, u); Ok(u) @@ -618,7 +619,8 @@ fn consts( match c.val { ty::ConstKind::Infer(InferConst::Var(vid)) => { - let variable_table = &mut self.infcx.inner.borrow_mut().const_unification_table; + let mut inner = self.infcx.inner.borrow_mut(); + let variable_table = &mut inner.const_unification_table(); let var_value = variable_table.probe_value(vid); match var_value.val { ConstVariableValue::Known { value: u } => self.relate(&u, &u), diff --git a/src/librustc_infer/infer/equate.rs b/src/librustc_infer/infer/equate.rs index e05094cda27..d054070e292 100644 --- a/src/librustc_infer/infer/equate.rs +++ b/src/librustc_infer/infer/equate.rs @@ -72,14 +72,14 @@ fn tys(&mut self, a: Ty<'tcx>, b: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> { } let infcx = self.fields.infcx; - let a = infcx.inner.borrow_mut().type_variables.replace_if_possible(a); - let b = infcx.inner.borrow_mut().type_variables.replace_if_possible(b); + let a = infcx.inner.borrow_mut().type_variables().replace_if_possible(a); + let b = infcx.inner.borrow_mut().type_variables().replace_if_possible(b); debug!("{}.tys: replacements ({:?}, {:?})", self.tag(), a, b); match (&a.kind, &b.kind) { (&ty::Infer(TyVar(a_id)), &ty::Infer(TyVar(b_id))) => { - infcx.inner.borrow_mut().type_variables.equate(a_id, b_id); + infcx.inner.borrow_mut().type_variables().equate(a_id, b_id); } (&ty::Infer(TyVar(a_id)), _) => { diff --git a/src/librustc_infer/infer/error_reporting/need_type_info.rs b/src/librustc_infer/infer/error_reporting/need_type_info.rs index 93c8e505697..d8133c58df7 100644 --- a/src/librustc_infer/infer/error_reporting/need_type_info.rs +++ b/src/librustc_infer/infer/error_reporting/need_type_info.rs @@ -59,7 +59,7 @@ fn node_ty_contains_target(&mut self, hir_id: HirId) -> Option> { .infcx .inner .borrow_mut() - .type_variables + .type_variables() .sub_unified(a_vid, b_vid), _ => false, } @@ -194,7 +194,8 @@ pub fn extract_type_name( highlight: Option, ) -> (String, Option, Cow<'static, str>, Option, Option<&'static str>) { if let ty::Infer(ty::TyVar(ty_vid)) = ty.kind { - let ty_vars = &self.inner.borrow().type_variables; + let mut inner = self.inner.borrow_mut(); + let ty_vars = &inner.type_variables(); let var_origin = ty_vars.var_origin(ty_vid); if let TypeVariableOriginKind::TypeParameterDefinition(name, def_id) = var_origin.kind { let parent_def_id = def_id.and_then(|def_id| self.tcx.parent(def_id)); @@ -248,7 +249,8 @@ pub fn need_type_info_err( let ty_to_string = |ty: Ty<'tcx>| -> String { let mut s = String::new(); let mut printer = ty::print::FmtPrinter::new(self.tcx, &mut s, Namespace::TypeNS); - let ty_vars = &self.inner.borrow().type_variables; + let mut inner = self.inner.borrow_mut(); + let ty_vars = inner.type_variables(); let getter = move |ty_vid| { let var_origin = ty_vars.var_origin(ty_vid); if let TypeVariableOriginKind::TypeParameterDefinition(name, _) = var_origin.kind { diff --git a/src/librustc_infer/infer/freshen.rs b/src/librustc_infer/infer/freshen.rs index 636cf42198b..47346c3a856 100644 --- a/src/librustc_infer/infer/freshen.rs +++ b/src/librustc_infer/infer/freshen.rs @@ -147,7 +147,7 @@ fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> { match t.kind { ty::Infer(ty::TyVar(v)) => { - let opt_ty = self.infcx.inner.borrow_mut().type_variables.probe(v).known(); + let opt_ty = self.infcx.inner.borrow_mut().type_variables().probe(v).known(); self.freshen_ty(opt_ty, ty::TyVar(v), ty::FreshTy) } @@ -155,7 +155,7 @@ fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> { self.infcx .inner .borrow_mut() - .int_unification_table + .int_unification_table() .probe_value(v) .map(|v| v.to_type(tcx)), ty::IntVar(v), @@ -166,7 +166,7 @@ fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> { self.infcx .inner .borrow_mut() - .float_unification_table + .float_unification_table() .probe_value(v) .map(|v| v.to_type(tcx)), ty::FloatVar(v), @@ -222,7 +222,7 @@ fn fold_const(&mut self, ct: &'tcx ty::Const<'tcx>) -> &'tcx ty::Const<'tcx> { .infcx .inner .borrow_mut() - .const_unification_table + .const_unification_table() .probe_value(v) .val .known(); diff --git a/src/librustc_infer/infer/fudge.rs b/src/librustc_infer/infer/fudge.rs index 1a58e100fb3..0046dba0b04 100644 --- a/src/librustc_infer/infer/fudge.rs +++ b/src/librustc_infer/infer/fudge.rs @@ -3,18 +3,30 @@ use super::type_variable::TypeVariableOrigin; use super::InferCtxt; -use super::{ConstVariableOrigin, RegionVariableOrigin}; +use super::{ConstVariableOrigin, RegionVariableOrigin, UnificationTable}; +use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::unify as ut; use ut::UnifyKey; use std::ops::Range; +fn vars_since_snapshot<'tcx, T>( + table: &mut UnificationTable<'_, 'tcx, T>, + snapshot: usize, +) -> Range +where + T: UnifyKey, + super::UndoLog<'tcx>: From>>, +{ + T::from_index(snapshot as u32)..T::from_index(table.len() as u32) +} + fn const_vars_since_snapshot<'tcx>( - table: &mut ut::UnificationTable>>, - snapshot: &ut::Snapshot>>, + table: &mut UnificationTable<'_, 'tcx, ConstVid<'tcx>>, + snapshot: usize, ) -> (Range>, Vec) { - let range = table.vars_since_snapshot(snapshot); + let range = vars_since_snapshot(table, snapshot); ( range.start..range.end, (range.start.index..range.end.index) @@ -83,17 +95,21 @@ pub fn fudge_inference_if_ok(&self, f: F) -> Result let mut inner = self.inner.borrow_mut(); let type_vars = - inner.type_variables.vars_since_snapshot(&snapshot.type_snapshot); - let int_vars = - inner.int_unification_table.vars_since_snapshot(&snapshot.int_snapshot); - let float_vars = - inner.float_unification_table.vars_since_snapshot(&snapshot.float_snapshot); + inner.type_variables().vars_since_snapshot(&snapshot.type_snapshot); + let int_vars = vars_since_snapshot( + &mut inner.int_unification_table(), + snapshot.int_snapshot, + ); + let float_vars = vars_since_snapshot( + &mut inner.float_unification_table(), + snapshot.float_snapshot, + ); let region_vars = inner .unwrap_region_constraints() .vars_since_snapshot(&snapshot.region_constraints_snapshot); let const_vars = const_vars_since_snapshot( - &mut inner.const_unification_table, - &snapshot.const_snapshot, + &mut inner.const_unification_table(), + snapshot.const_snapshot, ); let fudger = InferenceFudger { @@ -161,7 +177,7 @@ fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> { // that it is unbound, so we can just return // it. debug_assert!( - self.infcx.inner.borrow_mut().type_variables.probe(vid).is_unknown() + self.infcx.inner.borrow_mut().type_variables().probe(vid).is_unknown() ); ty } diff --git a/src/librustc_infer/infer/lattice.rs b/src/librustc_infer/infer/lattice.rs index c29614b8556..1bf43e74dcd 100644 --- a/src/librustc_infer/infer/lattice.rs +++ b/src/librustc_infer/infer/lattice.rs @@ -56,8 +56,8 @@ pub fn super_lattice_tys<'a, 'tcx: 'a, L>( } let infcx = this.infcx(); - let a = infcx.inner.borrow_mut().type_variables.replace_if_possible(a); - let b = infcx.inner.borrow_mut().type_variables.replace_if_possible(b); + let a = infcx.inner.borrow_mut().type_variables().replace_if_possible(a); + let b = infcx.inner.borrow_mut().type_variables().replace_if_possible(b); match (&a.kind, &b.kind) { // If one side is known to be a variable and one is not, // create a variable (`v`) to represent the LUB. Make sure to diff --git a/src/librustc_infer/infer/mod.rs b/src/librustc_infer/infer/mod.rs index 54f80e8f388..6d76f15998a 100644 --- a/src/librustc_infer/infer/mod.rs +++ b/src/librustc_infer/infer/mod.rs @@ -10,7 +10,9 @@ use rustc_ast::ast; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::sync::Lrc; +use rustc_data_structures::undo_log::{Rollback, Snapshots, UndoLogs}; use rustc_data_structures::unify as ut; use rustc_errors::DiagnosticBuilder; use rustc_hir as hir; @@ -36,6 +38,7 @@ use std::cell::{Cell, Ref, RefCell}; use std::collections::BTreeMap; use std::fmt; +use std::marker::PhantomData; use self::combine::CombineFields; use self::free_regions::RegionRelations; @@ -141,16 +144,17 @@ pub struct InferCtxtInner<'tcx> { /// We instantiate `UnificationTable` with `bounds` because the types /// that might instantiate a general type variable have an order, /// represented by its upper and lower bounds. - type_variables: type_variable::TypeVariableTable<'tcx>, + type_variables: type_variable::TypeVariableStorage<'tcx>, + undo_log: Logs<'tcx>, /// Map from const parameter variable to the kind of const it represents. - const_unification_table: ut::UnificationTable>>, + const_unification_table: ut::UnificationStorage>, /// Map from integral variable to the kind of integer it represents. - int_unification_table: ut::UnificationTable>, + int_unification_table: ut::UnificationStorage, /// Map from floating variable to the kind of float it represents. - float_unification_table: ut::UnificationTable>, + float_unification_table: ut::UnificationStorage, /// Tracks the set of region variables and the constraints between them. /// This is initially `Some(_)` but when @@ -197,20 +201,220 @@ impl<'tcx> InferCtxtInner<'tcx> { fn new() -> InferCtxtInner<'tcx> { InferCtxtInner { projection_cache: Default::default(), - type_variables: type_variable::TypeVariableTable::new(), - const_unification_table: ut::UnificationTable::new(), - int_unification_table: ut::UnificationTable::new(), - float_unification_table: ut::UnificationTable::new(), + type_variables: type_variable::TypeVariableStorage::new(), + undo_log: Logs::default(), + const_unification_table: ut::UnificationStorage::new(), + int_unification_table: ut::UnificationStorage::new(), + float_unification_table: ut::UnificationStorage::new(), region_constraints: Some(RegionConstraintCollector::new()), region_obligations: vec![], } } + fn type_variables(&mut self) -> type_variable::TypeVariableTable<'tcx, '_> { + self.type_variables.with_log(&mut self.undo_log) + } + + fn int_unification_table( + &mut self, + ) -> ut::UnificationTable< + ut::InPlace, &mut Logs<'tcx>>, + > { + ut::UnificationTable::with_log(&mut self.int_unification_table, &mut self.undo_log) + } + + fn float_unification_table( + &mut self, + ) -> ut::UnificationTable< + ut::InPlace, &mut Logs<'tcx>>, + > { + ut::UnificationTable::with_log(&mut self.float_unification_table, &mut self.undo_log) + } + + fn const_unification_table( + &mut self, + ) -> ut::UnificationTable< + ut::InPlace< + ty::ConstVid<'tcx>, + &mut ut::UnificationStorage>, + &mut Logs<'tcx>, + >, + > { + ut::UnificationTable::with_log(&mut self.const_unification_table, &mut self.undo_log) + } + pub fn unwrap_region_constraints(&mut self) -> &mut RegionConstraintCollector<'tcx> { self.region_constraints.as_mut().expect("region constraints already solved") } } +pub struct Snapshot<'tcx> { + undo_len: usize, + _marker: PhantomData<&'tcx ()>, +} + +pub(crate) enum UndoLog<'tcx> { + TypeVariables(type_variable::UndoLog<'tcx>), + ConstUnificationTable(sv::UndoLog>>), + IntUnificationTable(sv::UndoLog>), + FloatUnificationTable(sv::UndoLog>), +} + +impl<'tcx> From>>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>>) -> Self { + UndoLog::TypeVariables(type_variable::UndoLog::EqRelation(l)) + } +} + +impl<'tcx> From>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>) -> Self { + UndoLog::TypeVariables(type_variable::UndoLog::SubRelation(l)) + } +} + +impl<'tcx> From> for UndoLog<'tcx> { + fn from(l: sv::UndoLog) -> Self { + UndoLog::TypeVariables(type_variable::UndoLog::Values(l)) + } +} + +impl<'tcx> From for UndoLog<'tcx> { + fn from(l: type_variable::Instantiate) -> Self { + UndoLog::TypeVariables(type_variable::UndoLog::from(l)) + } +} + +impl From> for UndoLog<'tcx> { + fn from(t: type_variable::UndoLog<'tcx>) -> Self { + Self::TypeVariables(t) + } +} + +impl<'tcx> From>>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>>) -> Self { + Self::ConstUnificationTable(l) + } +} + +impl<'tcx> From>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>) -> Self { + Self::IntUnificationTable(l) + } +} + +impl<'tcx> From>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>) -> Self { + Self::FloatUnificationTable(l) + } +} + +pub(crate) type UnificationTable<'a, 'tcx, T> = + ut::UnificationTable, &'a mut Logs<'tcx>>>; + +struct RollbackView<'tcx, 'a> { + type_variables: type_variable::RollbackView<'tcx, 'a>, + const_unification_table: &'a mut ut::UnificationStorage>, + int_unification_table: &'a mut ut::UnificationStorage, + float_unification_table: &'a mut ut::UnificationStorage, +} + +impl<'tcx> Rollback> for RollbackView<'tcx, '_> { + fn reverse(&mut self, undo: UndoLog<'tcx>) { + match undo { + UndoLog::TypeVariables(undo) => self.type_variables.reverse(undo), + UndoLog::ConstUnificationTable(undo) => self.const_unification_table.reverse(undo), + UndoLog::IntUnificationTable(undo) => self.int_unification_table.reverse(undo), + UndoLog::FloatUnificationTable(undo) => self.float_unification_table.reverse(undo), + } + } +} + +pub(crate) struct Logs<'tcx> { + logs: Vec>, + num_open_snapshots: usize, +} + +impl Default for Logs<'_> { + fn default() -> Self { + Self { logs: Default::default(), num_open_snapshots: Default::default() } + } +} + +impl<'tcx, T> UndoLogs for Logs<'tcx> +where + UndoLog<'tcx>: From, +{ + fn num_open_snapshots(&self) -> usize { + self.num_open_snapshots + } + fn push(&mut self, undo: T) { + if self.in_snapshot() { + self.logs.push(undo.into()) + } + } + fn extend(&mut self, undos: J) + where + Self: Sized, + J: IntoIterator, + { + if self.in_snapshot() { + self.logs.extend(undos.into_iter().map(UndoLog::from)) + } + } +} + +impl<'tcx> Snapshots> for Logs<'tcx> { + type Snapshot = Snapshot<'tcx>; + fn actions_since_snapshot(&self, snapshot: &Self::Snapshot) -> &[UndoLog<'tcx>] { + &self.logs[snapshot.undo_len..] + } + + fn start_snapshot(&mut self) -> Self::Snapshot { + unreachable!() + } + + fn rollback_to(&mut self, values: &mut impl Rollback>, snapshot: Self::Snapshot) { + debug!("rollback_to({})", snapshot.undo_len); + self.assert_open_snapshot(&snapshot); + + while self.logs.len() > snapshot.undo_len { + values.reverse(self.logs.pop().unwrap()); + } + + if self.num_open_snapshots == 1 { + // The root snapshot. It's safe to clear the undo log because + // there's no snapshot further out that we might need to roll back + // to. + assert!(snapshot.undo_len == 0); + self.logs.clear(); + } + + self.num_open_snapshots -= 1; + } + + fn commit(&mut self, snapshot: Self::Snapshot) { + debug!("commit({})", snapshot.undo_len); + + if self.num_open_snapshots == 1 { + // The root snapshot. It's safe to clear the undo log because + // there's no snapshot further out that we might need to roll back + // to. + assert!(snapshot.undo_len == 0); + self.logs.clear(); + } + + self.num_open_snapshots -= 1; + } +} + +impl<'tcx> Logs<'tcx> { + fn assert_open_snapshot(&self, snapshot: &Snapshot<'tcx>) { + // Failures here may indicate a failure to follow a stack discipline. + assert!(self.logs.len() >= snapshot.undo_len); + assert!(self.num_open_snapshots > 0); + } +} + pub struct InferCtxt<'a, 'tcx> { pub tcx: TyCtxt<'tcx>, @@ -644,10 +848,11 @@ pub fn into_obligations(self) -> PredicateObligations<'tcx> { #[must_use = "once you start a snapshot, you should always consume it"] pub struct CombinedSnapshot<'a, 'tcx> { projection_cache_snapshot: traits::ProjectionCacheSnapshot, + undo_snapshot: Snapshot<'tcx>, type_snapshot: type_variable::Snapshot<'tcx>, - const_snapshot: ut::Snapshot>>, - int_snapshot: ut::Snapshot>, - float_snapshot: ut::Snapshot>, + const_snapshot: usize, + int_snapshot: usize, + float_snapshot: usize, region_constraints_snapshot: RegionSnapshot, region_obligations_snapshot: usize, universe: ty::UniverseIndex, @@ -667,7 +872,7 @@ pub fn freshen>(&self, t: T) -> T { pub fn type_var_diverges(&'a self, ty: Ty<'_>) -> bool { match ty.kind { - ty::Infer(ty::TyVar(vid)) => self.inner.borrow().type_variables.var_diverges(vid), + ty::Infer(ty::TyVar(vid)) => self.inner.borrow_mut().type_variables().var_diverges(vid), _ => false, } } @@ -681,14 +886,14 @@ pub fn type_is_unconstrained_numeric(&'a self, ty: Ty<'_>) -> UnconstrainedNumer use rustc_middle::ty::error::UnconstrainedNumeric::{UnconstrainedFloat, UnconstrainedInt}; match ty.kind { ty::Infer(ty::IntVar(vid)) => { - if self.inner.borrow_mut().int_unification_table.probe_value(vid).is_some() { + if self.inner.borrow_mut().int_unification_table().probe_value(vid).is_some() { Neither } else { UnconstrainedInt } } ty::Infer(ty::FloatVar(vid)) => { - if self.inner.borrow_mut().float_unification_table.probe_value(vid).is_some() { + if self.inner.borrow_mut().float_unification_table().probe_value(vid).is_some() { Neither } else { UnconstrainedFloat @@ -703,21 +908,21 @@ pub fn unsolved_variables(&self) -> Vec> { // FIXME(const_generics): should there be an equivalent function for const variables? let mut vars: Vec> = inner - .type_variables + .type_variables() .unsolved_variables() .into_iter() .map(|t| self.tcx.mk_ty_var(t)) .collect(); vars.extend( - (0..inner.int_unification_table.len()) + (0..inner.int_unification_table().len()) .map(|i| ty::IntVid { index: i as u32 }) - .filter(|&vid| inner.int_unification_table.probe_value(vid).is_none()) + .filter(|&vid| inner.int_unification_table().probe_value(vid).is_none()) .map(|v| self.tcx.mk_int_var(v)), ); vars.extend( - (0..inner.float_unification_table.len()) + (0..inner.float_unification_table().len()) .map(|i| ty::FloatVid { index: i as u32 }) - .filter(|&vid| inner.float_unification_table.probe_value(vid).is_none()) + .filter(|&vid| inner.float_unification_table().probe_value(vid).is_none()) .map(|v| self.tcx.mk_float_var(v)), ); vars @@ -769,12 +974,16 @@ fn start_snapshot(&self) -> CombinedSnapshot<'a, 'tcx> { let in_snapshot = self.in_snapshot.replace(true); let mut inner = self.inner.borrow_mut(); + + inner.undo_log.num_open_snapshots += 1; + let undo_snapshot = Snapshot { undo_len: inner.undo_log.logs.len(), _marker: PhantomData }; CombinedSnapshot { projection_cache_snapshot: inner.projection_cache.snapshot(), - type_snapshot: inner.type_variables.snapshot(), - const_snapshot: inner.const_unification_table.snapshot(), - int_snapshot: inner.int_unification_table.snapshot(), - float_snapshot: inner.float_unification_table.snapshot(), + undo_snapshot, + type_snapshot: inner.type_variables().snapshot(), + const_snapshot: inner.const_unification_table().len(), + int_snapshot: inner.int_unification_table().len(), + float_snapshot: inner.float_unification_table().len(), region_constraints_snapshot: inner.unwrap_region_constraints().start_snapshot(), region_obligations_snapshot: inner.region_obligations.len(), universe: self.universe(), @@ -790,10 +999,11 @@ fn rollback_to(&self, cause: &str, snapshot: CombinedSnapshot<'a, 'tcx>) { debug!("rollback_to(cause={})", cause); let CombinedSnapshot { projection_cache_snapshot, - type_snapshot, - const_snapshot, - int_snapshot, - float_snapshot, + undo_snapshot, + type_snapshot: _, + const_snapshot: _, + int_snapshot: _, + float_snapshot: _, region_constraints_snapshot, region_obligations_snapshot, universe, @@ -807,11 +1017,24 @@ fn rollback_to(&self, cause: &str, snapshot: CombinedSnapshot<'a, 'tcx>) { self.skip_leak_check.set(was_skip_leak_check); let mut inner = self.inner.borrow_mut(); + let inner = &mut *inner; + let InferCtxtInner { + type_variables, + const_unification_table, + int_unification_table, + float_unification_table, + .. + } = inner; + inner.undo_log.rollback_to( + &mut RollbackView { + type_variables: type_variable::RollbackView::from(type_variables), + const_unification_table, + int_unification_table, + float_unification_table, + }, + undo_snapshot, + ); inner.projection_cache.rollback_to(projection_cache_snapshot); - inner.type_variables.rollback_to(type_snapshot); - inner.const_unification_table.rollback_to(const_snapshot); - inner.int_unification_table.rollback_to(int_snapshot); - inner.float_unification_table.rollback_to(float_snapshot); inner.unwrap_region_constraints().rollback_to(region_constraints_snapshot); inner.region_obligations.truncate(region_obligations_snapshot); } @@ -820,10 +1043,11 @@ fn commit_from(&self, snapshot: CombinedSnapshot<'a, 'tcx>) { debug!("commit_from()"); let CombinedSnapshot { projection_cache_snapshot, - type_snapshot, - const_snapshot, - int_snapshot, - float_snapshot, + undo_snapshot, + type_snapshot: _, + const_snapshot: _, + int_snapshot: _, + float_snapshot: _, region_constraints_snapshot, region_obligations_snapshot: _, universe: _, @@ -836,11 +1060,8 @@ fn commit_from(&self, snapshot: CombinedSnapshot<'a, 'tcx>) { self.skip_leak_check.set(was_skip_leak_check); let mut inner = self.inner.borrow_mut(); + inner.undo_log.commit(undo_snapshot); inner.projection_cache.commit(projection_cache_snapshot); - inner.type_variables.commit(type_snapshot); - inner.const_unification_table.commit(const_snapshot); - inner.int_unification_table.commit(int_snapshot); - inner.float_unification_table.commit(float_snapshot); inner.unwrap_region_constraints().commit(region_constraints_snapshot); } @@ -1032,7 +1253,7 @@ pub fn region_outlives_predicate( } pub fn next_ty_var_id(&self, diverging: bool, origin: TypeVariableOrigin) -> TyVid { - self.inner.borrow_mut().type_variables.new_var(self.universe(), diverging, origin) + self.inner.borrow_mut().type_variables().new_var(self.universe(), diverging, origin) } pub fn next_ty_var(&self, origin: TypeVariableOrigin) -> Ty<'tcx> { @@ -1044,7 +1265,7 @@ pub fn next_ty_var_in_universe( origin: TypeVariableOrigin, universe: ty::UniverseIndex, ) -> Ty<'tcx> { - let vid = self.inner.borrow_mut().type_variables.new_var(universe, false, origin); + let vid = self.inner.borrow_mut().type_variables().new_var(universe, false, origin); self.tcx.mk_ty_var(vid) } @@ -1069,20 +1290,20 @@ pub fn next_const_var_in_universe( let vid = self .inner .borrow_mut() - .const_unification_table + .const_unification_table() .new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe } }); self.tcx.mk_const_var(vid, ty) } pub fn next_const_var_id(&self, origin: ConstVariableOrigin) -> ConstVid<'tcx> { - self.inner.borrow_mut().const_unification_table.new_key(ConstVarValue { + self.inner.borrow_mut().const_unification_table().new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe: self.universe() }, }) } fn next_int_var_id(&self) -> IntVid { - self.inner.borrow_mut().int_unification_table.new_key(None) + self.inner.borrow_mut().int_unification_table().new_key(None) } pub fn next_int_var(&self) -> Ty<'tcx> { @@ -1090,7 +1311,7 @@ pub fn next_int_var(&self) -> Ty<'tcx> { } fn next_float_var_id(&self) -> FloatVid { - self.inner.borrow_mut().float_unification_table.new_key(None) + self.inner.borrow_mut().float_unification_table().new_key(None) } pub fn next_float_var(&self) -> Ty<'tcx> { @@ -1161,7 +1382,7 @@ pub fn var_for_def(&self, span: Span, param: &ty::GenericParamDef) -> GenericArg // used in a path such as `Foo::::new()` will // use an inference variable for `C` with `[T, U]` // as the substitutions for the default, `(T, U)`. - let ty_var_id = self.inner.borrow_mut().type_variables.new_var( + let ty_var_id = self.inner.borrow_mut().type_variables().new_var( self.universe(), false, TypeVariableOrigin { @@ -1181,7 +1402,7 @@ pub fn var_for_def(&self, span: Span, param: &ty::GenericParamDef) -> GenericArg span, }; let const_var_id = - self.inner.borrow_mut().const_unification_table.new_key(ConstVarValue { + self.inner.borrow_mut().const_unification_table().new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe: self.universe() }, }); @@ -1335,7 +1556,7 @@ pub fn trait_ref_to_string(&self, t: &ty::TraitRef<'tcx>) -> String { pub fn probe_ty_var(&self, vid: TyVid) -> Result, ty::UniverseIndex> { use self::type_variable::TypeVariableValue; - match self.inner.borrow_mut().type_variables.probe(vid) { + match self.inner.borrow_mut().type_variables().probe(vid) { TypeVariableValue::Known { value } => Ok(value), TypeVariableValue::Unknown { universe } => Err(universe), } @@ -1357,7 +1578,7 @@ pub fn shallow_resolve(&self, value: T) -> T } pub fn root_var(&self, var: ty::TyVid) -> ty::TyVid { - self.inner.borrow_mut().type_variables.root_var(var) + self.inner.borrow_mut().type_variables().root_var(var) } /// Where possible, replaces type/const variables in @@ -1395,7 +1616,7 @@ pub fn probe_const_var( &self, vid: ty::ConstVid<'tcx>, ) -> Result<&'tcx ty::Const<'tcx>, ty::UniverseIndex> { - match self.inner.borrow_mut().const_unification_table.probe_value(vid).val { + match self.inner.borrow_mut().const_unification_table().probe_value(vid).val { ConstVariableValue::Known { value } => Ok(value), ConstVariableValue::Unknown { universe } => Err(universe), } @@ -1576,14 +1797,14 @@ fn shallow_resolve_ty(&self, typ: Ty<'tcx>) -> Ty<'tcx> { // // Note: if these two lines are combined into one we get // dynamic borrow errors on `self.inner`. - let known = self.inner.borrow_mut().type_variables.probe(v).known(); + let known = self.inner.borrow_mut().type_variables().probe(v).known(); known.map(|t| self.shallow_resolve_ty(t)).unwrap_or(typ) } ty::Infer(ty::IntVar(v)) => self .inner .borrow_mut() - .int_unification_table + .int_unification_table() .probe_value(v) .map(|v| v.to_type(self.tcx)) .unwrap_or(typ), @@ -1591,7 +1812,7 @@ fn shallow_resolve_ty(&self, typ: Ty<'tcx>) -> Ty<'tcx> { ty::Infer(ty::FloatVar(v)) => self .inner .borrow_mut() - .float_unification_table + .float_unification_table() .probe_value(v) .map(|v| v.to_type(self.tcx)) .unwrap_or(typ), @@ -1617,7 +1838,7 @@ pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar<'tcx>) // If `inlined_probe` returns a `Known` value, it never equals // `ty::Infer(ty::TyVar(v))`. - match self.inner.borrow_mut().type_variables.inlined_probe(v) { + match self.inner.borrow_mut().type_variables().inlined_probe(v) { TypeVariableValue::Unknown { .. } => false, TypeVariableValue::Known { .. } => true, } @@ -1627,7 +1848,7 @@ pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar<'tcx>) // If `inlined_probe_value` returns a value it's always a // `ty::Int(_)` or `ty::UInt(_)`, which never matches a // `ty::Infer(_)`. - self.inner.borrow_mut().int_unification_table.inlined_probe_value(v).is_some() + self.inner.borrow_mut().int_unification_table().inlined_probe_value(v).is_some() } TyOrConstInferVar::TyFloat(v) => { @@ -1635,7 +1856,7 @@ pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar<'tcx>) // `ty::Float(_)`, which never matches a `ty::Infer(_)`. // // Not `inlined_probe_value(v)` because this call site is colder. - self.inner.borrow_mut().float_unification_table.probe_value(v).is_some() + self.inner.borrow_mut().float_unification_table().probe_value(v).is_some() } TyOrConstInferVar::Const(v) => { @@ -1718,7 +1939,7 @@ fn fold_const(&mut self, ct: &'tcx ty::Const<'tcx>) -> &'tcx ty::Const<'tcx> { self.infcx .inner .borrow_mut() - .const_unification_table + .const_unification_table() .probe_value(*vid) .val .known() diff --git a/src/librustc_infer/infer/nll_relate/mod.rs b/src/librustc_infer/infer/nll_relate/mod.rs index a2907e6e373..7aea26987a2 100644 --- a/src/librustc_infer/infer/nll_relate/mod.rs +++ b/src/librustc_infer/infer/nll_relate/mod.rs @@ -311,7 +311,7 @@ fn relate_ty_var>( match value_ty.kind { ty::Infer(ty::TyVar(value_vid)) => { // Two type variables: just equate them. - self.infcx.inner.borrow_mut().type_variables.equate(vid, value_vid); + self.infcx.inner.borrow_mut().type_variables().equate(vid, value_vid); return Ok(value_ty); } @@ -332,7 +332,7 @@ fn relate_ty_var>( assert!(!generalized_ty.has_infer_types_or_consts()); } - self.infcx.inner.borrow_mut().type_variables.instantiate(vid, generalized_ty); + self.infcx.inner.borrow_mut().type_variables().instantiate(vid, generalized_ty); // The generalized values we extract from `canonical_var_values` have // been fully instantiated and hence the set of scopes we have @@ -362,7 +362,7 @@ fn generalize_value>( delegate: &mut self.delegate, first_free_index: ty::INNERMOST, ambient_variance: self.ambient_variance, - for_vid_sub_root: self.infcx.inner.borrow_mut().type_variables.sub_root_var(for_vid), + for_vid_sub_root: self.infcx.inner.borrow_mut().type_variables().sub_root_var(for_vid), universe, }; @@ -859,7 +859,8 @@ fn tys(&mut self, a: Ty<'tcx>, _: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> { } ty::Infer(ty::TyVar(vid)) => { - let variables = &mut self.infcx.inner.borrow_mut().type_variables; + let mut inner = self.infcx.inner.borrow_mut(); + let variables = &mut inner.type_variables(); let vid = variables.root_var(vid); let sub_vid = variables.sub_root_var(vid); if sub_vid == self.for_vid_sub_root { @@ -961,7 +962,8 @@ fn consts( bug!("unexpected inference variable encountered in NLL generalization: {:?}", a); } ty::ConstKind::Infer(InferConst::Var(vid)) => { - let variable_table = &mut self.infcx.inner.borrow_mut().const_unification_table; + let mut inner = self.infcx.inner.borrow_mut(); + let variable_table = &mut inner.const_unification_table(); let var_value = variable_table.probe_value(vid); match var_value.val.known() { Some(u) => self.relate(&u, &u), diff --git a/src/librustc_infer/infer/resolve.rs b/src/librustc_infer/infer/resolve.rs index bd9d108cfe8..e28cf49c7f2 100644 --- a/src/librustc_infer/infer/resolve.rs +++ b/src/librustc_infer/infer/resolve.rs @@ -123,7 +123,8 @@ fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { // Since we called `shallow_resolve` above, this must // be an (as yet...) unresolved inference variable. let ty_var_span = if let ty::TyVar(ty_vid) = infer_ty { - let ty_vars = &self.infcx.inner.borrow().type_variables; + let mut inner = self.infcx.inner.borrow_mut(); + let ty_vars = &inner.type_variables(); if let TypeVariableOrigin { kind: TypeVariableOriginKind::TypeParameterDefinition(_, _), span, diff --git a/src/librustc_infer/infer/sub.rs b/src/librustc_infer/infer/sub.rs index 080af37492d..0abcc15d6fc 100644 --- a/src/librustc_infer/infer/sub.rs +++ b/src/librustc_infer/infer/sub.rs @@ -80,8 +80,8 @@ fn tys(&mut self, a: Ty<'tcx>, b: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> { } let infcx = self.fields.infcx; - let a = infcx.inner.borrow_mut().type_variables.replace_if_possible(a); - let b = infcx.inner.borrow_mut().type_variables.replace_if_possible(b); + let a = infcx.inner.borrow_mut().type_variables().replace_if_possible(a); + let b = infcx.inner.borrow_mut().type_variables().replace_if_possible(b); match (&a.kind, &b.kind) { (&ty::Infer(TyVar(a_vid)), &ty::Infer(TyVar(b_vid))) => { // Shouldn't have any LBR here, so we can safely put @@ -95,7 +95,7 @@ fn tys(&mut self, a: Ty<'tcx>, b: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> { // have to record in the `type_variables` tracker that // the two variables are equal modulo subtyping, which // is important to the occurs check later on. - infcx.inner.borrow_mut().type_variables.sub(a_vid, b_vid); + infcx.inner.borrow_mut().type_variables().sub(a_vid, b_vid); self.fields.obligations.push(Obligation::new( self.fields.trace.cause.clone(), self.fields.param_env, diff --git a/src/librustc_infer/infer/type_variable.rs b/src/librustc_infer/infer/type_variable.rs index 1de820cdb62..47f7d764136 100644 --- a/src/librustc_infer/infer/type_variable.rs +++ b/src/librustc_infer/infer/type_variable.rs @@ -3,19 +3,76 @@ use rustc_span::symbol::Symbol; use rustc_span::Span; +use crate::infer::Logs; + use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::unify as ut; use std::cmp; use std::marker::PhantomData; use std::ops::Range; -pub struct TypeVariableTable<'tcx> { - values: sv::SnapshotVec, +use rustc_data_structures::undo_log::{Rollback, Snapshots, UndoLogs}; + +pub(crate) enum UndoLog<'tcx> { + EqRelation(sv::UndoLog>>), + SubRelation(sv::UndoLog>), + Values(sv::UndoLog), +} + +impl<'tcx> From>>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>>) -> Self { + UndoLog::EqRelation(l) + } +} + +impl<'tcx> From>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>) -> Self { + UndoLog::SubRelation(l) + } +} + +impl<'tcx> From> for UndoLog<'tcx> { + fn from(l: sv::UndoLog) -> Self { + UndoLog::Values(l) + } +} + +impl<'tcx> From for UndoLog<'tcx> { + fn from(l: Instantiate) -> Self { + UndoLog::Values(sv::UndoLog::Other(l)) + } +} + +pub(crate) struct RollbackView<'tcx, 'a> { + pub(crate) eq_relations: &'a mut ut::UnificationStorage>, + pub(crate) sub_relations: &'a mut ut::UnificationStorage, + pub(crate) values: &'a mut Vec, +} + +impl<'tcx, 'a> From<&'a mut TypeVariableStorage<'tcx>> for RollbackView<'tcx, 'a> { + fn from(l: &'a mut TypeVariableStorage<'tcx>) -> Self { + let TypeVariableStorage { eq_relations, sub_relations, values } = l; + Self { eq_relations, sub_relations, values } + } +} + +impl<'tcx> Rollback> for RollbackView<'tcx, '_> { + fn reverse(&mut self, undo: UndoLog<'tcx>) { + match undo { + UndoLog::EqRelation(undo) => self.eq_relations.reverse(undo), + UndoLog::SubRelation(undo) => self.sub_relations.reverse(undo), + UndoLog::Values(undo) => self.values.reverse(undo), + } + } +} + +pub struct TypeVariableStorage<'tcx> { + values: Vec, /// Two variables are unified in `eq_relations` when we have a /// constraint `?X == ?Y`. This table also stores, for each key, /// the known value. - eq_relations: ut::UnificationTable>>, + eq_relations: ut::UnificationStorage>, /// Two variables are unified in `sub_relations` when we have a /// constraint `?X <: ?Y` *or* a constraint `?Y <: ?X`. This second @@ -34,7 +91,17 @@ pub struct TypeVariableTable<'tcx> { /// This is reasonable because, in Rust, subtypes have the same /// "skeleton" and hence there is no possible type such that /// (e.g.) `Box <: ?3` for any `?3`. - sub_relations: ut::UnificationTable>, + sub_relations: ut::UnificationStorage, +} + +pub struct TypeVariableTable<'tcx, 'a> { + values: &'a mut Vec, + + eq_relations: &'a mut ut::UnificationStorage>, + + sub_relations: &'a mut ut::UnificationStorage, + + undo_log: &'a mut Logs<'tcx>, } #[derive(Copy, Clone, Debug)] @@ -62,7 +129,7 @@ pub enum TypeVariableOriginKind { LatticeVariable, } -struct TypeVariableData { +pub(crate) struct TypeVariableData { origin: TypeVariableOrigin, diverging: bool, } @@ -92,32 +159,41 @@ pub fn is_unknown(&self) -> bool { } pub struct Snapshot<'tcx> { - snapshot: sv::Snapshot, - eq_snapshot: ut::Snapshot>>, - sub_snapshot: ut::Snapshot>, + value_count: u32, + _marker: PhantomData<&'tcx ()>, } -struct Instantiate { +pub(crate) struct Instantiate { vid: ty::TyVid, } -struct Delegate; +pub(crate) struct Delegate; -impl<'tcx> TypeVariableTable<'tcx> { - pub fn new() -> TypeVariableTable<'tcx> { - TypeVariableTable { - values: sv::SnapshotVec::new(), - eq_relations: ut::UnificationTable::new(), - sub_relations: ut::UnificationTable::new(), +impl<'tcx> TypeVariableStorage<'tcx> { + pub fn new() -> TypeVariableStorage<'tcx> { + TypeVariableStorage { + values: Vec::new(), + eq_relations: ut::UnificationStorage::new(), + sub_relations: ut::UnificationStorage::new(), } } + pub(crate) fn with_log<'a>( + &'a mut self, + undo_log: &'a mut Logs<'tcx>, + ) -> TypeVariableTable<'tcx, 'a> { + let TypeVariableStorage { values, eq_relations, sub_relations } = self; + TypeVariableTable { values, eq_relations, sub_relations, undo_log } + } +} + +impl<'tcx> TypeVariableTable<'tcx, '_> { /// Returns the diverges flag given when `vid` was created. /// /// Note that this function does not return care whether /// `vid` has been unified with something else or not. pub fn var_diverges(&self, vid: ty::TyVid) -> bool { - self.values.get(vid.index as usize).diverging + self.values.get(vid.index as usize).unwrap().diverging } /// Returns the origin that was given when `vid` was created. @@ -125,7 +201,7 @@ pub fn var_diverges(&self, vid: ty::TyVid) -> bool { /// Note that this function does not return care whether /// `vid` has been unified with something else or not. pub fn var_origin(&self, vid: ty::TyVid) -> &TypeVariableOrigin { - &self.values.get(vid.index as usize).origin + &self.values.get(vid.index as usize).unwrap().origin } /// Records that `a == b`, depending on `dir`. @@ -134,8 +210,8 @@ pub fn var_origin(&self, vid: ty::TyVid) -> &TypeVariableOrigin { pub fn equate(&mut self, a: ty::TyVid, b: ty::TyVid) { debug_assert!(self.probe(a).is_unknown()); debug_assert!(self.probe(b).is_unknown()); - self.eq_relations.union(a, b); - self.sub_relations.union(a, b); + self.eq_relations().union(a, b); + self.sub_relations().union(a, b); } /// Records that `a <: b`, depending on `dir`. @@ -144,7 +220,7 @@ pub fn equate(&mut self, a: ty::TyVid, b: ty::TyVid) { pub fn sub(&mut self, a: ty::TyVid, b: ty::TyVid) { debug_assert!(self.probe(a).is_unknown()); debug_assert!(self.probe(b).is_unknown()); - self.sub_relations.union(a, b); + self.sub_relations().union(a, b); } /// Instantiates `vid` with the type `ty`. @@ -154,18 +230,18 @@ pub fn instantiate(&mut self, vid: ty::TyVid, ty: Ty<'tcx>) { let vid = self.root_var(vid); debug_assert!(self.probe(vid).is_unknown()); debug_assert!( - self.eq_relations.probe_value(vid).is_unknown(), + self.eq_relations().probe_value(vid).is_unknown(), "instantiating type variable `{:?}` twice: new-value = {:?}, old-value={:?}", vid, ty, - self.eq_relations.probe_value(vid) + self.eq_relations().probe_value(vid) ); - self.eq_relations.union_value(vid, TypeVariableValue::Known { value: ty }); + self.eq_relations().union_value(vid, TypeVariableValue::Known { value: ty }); // Hack: we only need this so that `types_escaping_snapshot` // can see what has been unified; see the Delegate impl for // more details. - self.values.record(Instantiate { vid }); + self.undo_log.push(Instantiate { vid }); } /// Creates a new type variable. @@ -184,12 +260,12 @@ pub fn new_var( diverging: bool, origin: TypeVariableOrigin, ) -> ty::TyVid { - let eq_key = self.eq_relations.new_key(TypeVariableValue::Unknown { universe }); + let eq_key = self.eq_relations().new_key(TypeVariableValue::Unknown { universe }); - let sub_key = self.sub_relations.new_key(()); + let sub_key = self.sub_relations().new_key(()); assert_eq!(eq_key.vid, sub_key); - let index = self.values.push(TypeVariableData { origin, diverging }); + let index = self.values().push(TypeVariableData { origin, diverging }); assert_eq!(eq_key.vid.index, index as u32); debug!( @@ -211,7 +287,7 @@ pub fn num_vars(&self) -> usize { /// algorithm), so `root_var(a) == root_var(b)` implies that `a == /// b` (transitively). pub fn root_var(&mut self, vid: ty::TyVid) -> ty::TyVid { - self.eq_relations.find(vid).vid + self.eq_relations().find(vid).vid } /// Returns the "root" variable of `vid` in the `sub_relations` @@ -222,7 +298,7 @@ pub fn root_var(&mut self, vid: ty::TyVid) -> ty::TyVid { /// /// exists X. (a <: X || X <: a) && (b <: X || X <: b) pub fn sub_root_var(&mut self, vid: ty::TyVid) -> ty::TyVid { - self.sub_relations.find(vid) + self.sub_relations().find(vid) } /// Returns `true` if `a` and `b` have same "sub-root" (i.e., exists some @@ -240,7 +316,7 @@ pub fn probe(&mut self, vid: ty::TyVid) -> TypeVariableValue<'tcx> { /// An always-inlined variant of `probe`, for very hot call sites. #[inline(always)] pub fn inlined_probe(&mut self, vid: ty::TyVid) -> TypeVariableValue<'tcx> { - self.eq_relations.inlined_probe_value(vid) + self.eq_relations().inlined_probe_value(vid) } /// If `t` is a type-inference variable, and it has been @@ -261,40 +337,31 @@ pub fn replace_if_possible(&mut self, t: Ty<'tcx>) -> Ty<'tcx> { /// (`rollback_to()`). Nested snapshots are permitted, but must /// be processed in a stack-like fashion. pub fn snapshot(&mut self) -> Snapshot<'tcx> { - Snapshot { - snapshot: self.values.start_snapshot(), - eq_snapshot: self.eq_relations.snapshot(), - sub_snapshot: self.sub_relations.snapshot(), - } + Snapshot { value_count: self.eq_relations().len() as u32, _marker: PhantomData } } - /// Undoes all changes since the snapshot was created. Any - /// snapshots created since that point must already have been - /// committed or rolled back. - pub fn rollback_to(&mut self, s: Snapshot<'tcx>) { - debug!("rollback_to{:?}", { - for action in self.values.actions_since_snapshot(&s.snapshot) { - if let sv::UndoLog::NewElem(index) = *action { - debug!("inference variable _#{}t popped", index) - } - } - }); + fn values(&mut self) -> sv::SnapshotVec, &mut Logs<'tcx>> { + sv::SnapshotVec::with_log(self.values, self.undo_log) + } - let Snapshot { snapshot, eq_snapshot, sub_snapshot } = s; - self.values.rollback_to(snapshot); - self.eq_relations.rollback_to(eq_snapshot); - self.sub_relations.rollback_to(sub_snapshot); + fn eq_relations( + &mut self, + ) -> ut::UnificationTable< + ut::InPlace< + TyVidEqKey<'tcx>, + &mut ut::UnificationStorage>, + &mut Logs<'tcx>, + >, + > { + ut::UnificationTable::with_log(self.eq_relations, self.undo_log) } - /// Commits all changes since the snapshot was created, making - /// them permanent (unless this snapshot was created within - /// another snapshot). Any snapshots created since that point - /// must already have been committed or rolled back. - pub fn commit(&mut self, s: Snapshot<'tcx>) { - let Snapshot { snapshot, eq_snapshot, sub_snapshot } = s; - self.values.commit(snapshot); - self.eq_relations.commit(eq_snapshot); - self.sub_relations.commit(sub_snapshot); + fn sub_relations( + &mut self, + ) -> ut::UnificationTable< + ut::InPlace, &mut Logs<'tcx>>, + > { + ut::UnificationTable::with_log(self.sub_relations, self.undo_log) } /// Returns a range of the type variables created during the snapshot. @@ -302,11 +369,12 @@ pub fn vars_since_snapshot( &mut self, s: &Snapshot<'tcx>, ) -> (Range, Vec) { - let range = self.eq_relations.vars_since_snapshot(&s.eq_snapshot); + let range = + TyVid { index: s.value_count }..TyVid { index: self.eq_relations().len() as u32 }; ( - range.start.vid..range.end.vid, - (range.start.vid.index..range.end.vid.index) - .map(|index| self.values.get(index as usize).origin) + range.start..range.end, + (range.start.index..range.end.index) + .map(|index| self.values.get(index as usize).unwrap().origin) .collect(), ) } @@ -317,14 +385,15 @@ pub fn vars_since_snapshot( /// a type variable `V0`, then we started the snapshot, then we /// created a type variable `V1`, unified `V0` with `T0`, and /// unified `V1` with `T1`, this function would return `{T0}`. - pub fn types_escaping_snapshot(&mut self, s: &Snapshot<'tcx>) -> Vec> { + pub fn types_escaping_snapshot(&mut self, s: &super::Snapshot<'tcx>) -> Vec> { let mut new_elem_threshold = u32::MAX; let mut escaping_types = Vec::new(); - let actions_since_snapshot = self.values.actions_since_snapshot(&s.snapshot); + let actions_since_snapshot = self.undo_log.actions_since_snapshot(s); debug!("actions_since_snapshot.len() = {}", actions_since_snapshot.len()); - for action in actions_since_snapshot { - match *action { - sv::UndoLog::NewElem(index) => { + for i in 0..actions_since_snapshot.len() { + let actions_since_snapshot = self.undo_log.actions_since_snapshot(s); + match actions_since_snapshot[i] { + super::UndoLog::TypeVariables(UndoLog::Values(sv::UndoLog::NewElem(index))) => { // if any new variables were created during the // snapshot, remember the lower index (which will // always be the first one we see). Note that this @@ -334,11 +403,17 @@ pub fn types_escaping_snapshot(&mut self, s: &Snapshot<'tcx>) -> Vec> { debug!("NewElem({}) new_elem_threshold={}", index, new_elem_threshold); } - sv::UndoLog::Other(Instantiate { vid, .. }) => { + super::UndoLog::TypeVariables(UndoLog::Values(sv::UndoLog::Other( + Instantiate { vid, .. }, + ))) => { if vid.index < new_elem_threshold { // quick check to see if this variable was // created since the snapshot started or not. - let escaping_type = match self.eq_relations.probe_value(vid) { + let mut eq_relations = ut::UnificationTable::with_log( + &mut *self.eq_relations, + &mut *self.undo_log, + ); + let escaping_type = match eq_relations.probe_value(vid) { TypeVariableValue::Unknown { .. } => bug!(), TypeVariableValue::Known { value } => value, }; @@ -395,7 +470,7 @@ fn reverse(_values: &mut Vec, _action: Instantiate) { /// for the `eq_relations`; they carry a `TypeVariableValue` along /// with them. #[derive(Copy, Clone, Debug, PartialEq, Eq)] -struct TyVidEqKey<'tcx> { +pub(crate) struct TyVidEqKey<'tcx> { vid: ty::TyVid, // in the table, we map each ty-vid to one of these: diff --git a/src/librustc_middle/infer/unify_key.rs b/src/librustc_middle/infer/unify_key.rs index e205453a48c..2580ac6bebd 100644 --- a/src/librustc_middle/infer/unify_key.rs +++ b/src/librustc_middle/infer/unify_key.rs @@ -1,6 +1,9 @@ use crate::ty::{self, FloatVarValue, InferConst, IntVarValue, Ty, TyCtxt}; -use rustc_data_structures::unify::InPlace; -use rustc_data_structures::unify::{EqUnifyValue, NoError, UnificationTable, UnifyKey, UnifyValue}; +use rustc_data_structures::snapshot_vec; +use rustc_data_structures::undo_log::UndoLogs; +use rustc_data_structures::unify::{ + self, EqUnifyValue, InPlace, NoError, UnificationTable, UnifyKey, UnifyValue, +}; use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; @@ -212,10 +215,14 @@ fn unify_values(value1: &Self, value2: &Self) -> Result { impl<'tcx> EqUnifyValue for &'tcx ty::Const<'tcx> {} -pub fn replace_if_possible( - table: &mut UnificationTable>>, +pub fn replace_if_possible( + table: &mut UnificationTable, V, L>>, c: &'tcx ty::Const<'tcx>, -) -> &'tcx ty::Const<'tcx> { +) -> &'tcx ty::Const<'tcx> +where + V: snapshot_vec::VecLike>>, + L: UndoLogs>>>, +{ if let ty::Const { val: ty::ConstKind::Infer(InferConst::Var(vid)), .. } = c { match table.probe_value(*vid).val.known() { Some(c) => c,