From b2f25e3c38ff29eebe6c8ce69b8c69243faa440d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 21 Sep 2018 20:42:49 +1000 Subject: [PATCH] Compress `Liveness` data some more. Profiling shows that the `(reader, writer, used)` triples used by liveness analysis almost always have invalid `reader` and `writer` fields. We can take advantage of this knowledge to use a compressed representation for them, falling back to a secondary table for the uncommon cases. This change reduces instruction counts on numerous benchmarks, the best by 16%. It also reduces max-rss on numerous benchmarks, the best by 38%. The patch also renames these triples from `Users` to `RWU`, because it's confusing having a type whose name is plural and then used within vectors whose names are also plural. --- src/librustc/middle/liveness.rs | 207 +++++++++++++++++++++++--------- 1 file changed, 149 insertions(+), 58 deletions(-) diff --git a/src/librustc/middle/liveness.rs b/src/librustc/middle/liveness.rs index 13847fb48ce..0d70a64123b 100644 --- a/src/librustc/middle/liveness.rs +++ b/src/librustc/middle/liveness.rs @@ -64,10 +64,10 @@ //! methods. It effectively does a reverse walk of the AST; whenever we //! reach a loop node, we iterate until a fixed point is reached. //! -//! ## The `users_*` fields +//! ## The `RWU` struct //! //! At each live node `N`, we track three pieces of information for each -//! variable `V` (these are in the `users_*` fields): +//! variable `V` (these are encapsulated in the `RWU` struct): //! //! - `reader`: the `LiveNode` ID of some node which will read the value //! that `V` holds on entry to `N`. Formally: a node `M` such @@ -536,6 +536,112 @@ fn visit_expr<'a, 'tcx>(ir: &mut IrMaps<'a, 'tcx>, expr: &'tcx Expr) { // Actually we compute just a bit more than just liveness, but we use // the same basic propagation framework in all cases. +#[derive(Clone, Copy)] +struct RWU { + reader: LiveNode, + writer: LiveNode, + used: bool +} + +/// Conceptually, this is like a `Vec`. But the number of `RWU`s can get +/// very large, so it uses a more compact representation that takes advantage +/// of the fact that when the number of `RWU`s is large, most of them have an +/// invalid reader and an invalid writer. +struct RWUTable { + /// Each entry in `packed_rwus` is either INV_INV_FALSE, INV_INV_TRUE, or + /// an index into `unpacked_rwus`. In the common cases, this compacts the + /// 65 bits of data into 32; in the uncommon cases, it expands the 65 bits + /// in 96. + /// + /// More compact representations are possible -- e.g. use only 2 bits per + /// packed `RWU` and make the secondary table a HashMap that maps from + /// indices to `RWU`s -- but this one strikes a good balance between size + /// and speed. + packed_rwus: Vec, + unpacked_rwus: Vec, +} + +// A constant representing `RWU { reader: invalid_node(); writer: invalid_node(); used: false }`. +const INV_INV_FALSE: u32 = u32::MAX; + +// A constant representing `RWU { reader: invalid_node(); writer: invalid_node(); used: true }`. +const INV_INV_TRUE: u32 = u32::MAX - 1; + +impl RWUTable { + fn new(num_rwus: usize) -> RWUTable { + Self { + packed_rwus: vec![INV_INV_FALSE; num_rwus], + unpacked_rwus: vec![], + } + } + + fn get(&self, idx: usize) -> RWU { + let packed_rwu = self.packed_rwus[idx]; + match packed_rwu { + INV_INV_FALSE => RWU { reader: invalid_node(), writer: invalid_node(), used: false }, + INV_INV_TRUE => RWU { reader: invalid_node(), writer: invalid_node(), used: true }, + _ => self.unpacked_rwus[packed_rwu as usize], + } + } + + fn get_reader(&self, idx: usize) -> LiveNode { + let packed_rwu = self.packed_rwus[idx]; + match packed_rwu { + INV_INV_FALSE | INV_INV_TRUE => invalid_node(), + _ => self.unpacked_rwus[packed_rwu as usize].reader, + } + } + + fn get_writer(&self, idx: usize) -> LiveNode { + let packed_rwu = self.packed_rwus[idx]; + match packed_rwu { + INV_INV_FALSE | INV_INV_TRUE => invalid_node(), + _ => self.unpacked_rwus[packed_rwu as usize].writer, + } + } + + fn get_used(&self, idx: usize) -> bool { + let packed_rwu = self.packed_rwus[idx]; + match packed_rwu { + INV_INV_FALSE => false, + INV_INV_TRUE => true, + _ => self.unpacked_rwus[packed_rwu as usize].used, + } + } + + #[inline] + fn copy_packed(&mut self, dst_idx: usize, src_idx: usize) { + self.packed_rwus[dst_idx] = self.packed_rwus[src_idx]; + } + + fn assign_unpacked(&mut self, idx: usize, rwu: RWU) { + if rwu.reader == invalid_node() && rwu.writer == invalid_node() { + // When we overwrite an indexing entry in `self.packed_rwus` with + // `INV_INV_{TRUE,FALSE}` we don't remove the corresponding entry + // from `self.unpacked_rwus`; it's not worth the effort, and we + // can't have entries shifting around anyway. + self.packed_rwus[idx] = if rwu.used { + INV_INV_TRUE + } else { + INV_INV_FALSE + } + } else { + // Add a new RWU to `unpacked_rwus` and make `packed_rwus[idx]` + // point to it. + self.packed_rwus[idx] = self.unpacked_rwus.len() as u32; + self.unpacked_rwus.push(rwu); + } + } + + fn assign_inv_inv(&mut self, idx: usize) { + self.packed_rwus[idx] = if self.get_used(idx) { + INV_INV_TRUE + } else { + INV_INV_FALSE + }; + } +} + #[derive(Copy, Clone)] struct Specials { exit_ln: LiveNode, @@ -552,14 +658,7 @@ struct Liveness<'a, 'tcx: 'a> { tables: &'a ty::TypeckTables<'tcx>, s: Specials, successors: Vec, - - // We used to have a single `users: Vec` field here, where `Users` - // had `reader`, `writer` and `used` fields. But the number of users can - // get very large, and it's more compact to store the data in three - // separate `Vec`s so that no space is wasted for padding. - users_reader: Vec, - users_writer: Vec, - users_used: Vec, + rwu_table: RWUTable, // mappings from loop node ID to LiveNode // ("break" label should map to loop node ID, @@ -584,16 +683,13 @@ fn new(ir: &'a mut IrMaps<'a, 'tcx>, body: hir::BodyId) -> Liveness<'a, 'tcx> { let num_live_nodes = ir.num_live_nodes; let num_vars = ir.num_vars; - let num_users = num_live_nodes * num_vars; Liveness { ir, tables, s: specials, successors: vec![invalid_node(); num_live_nodes], - users_reader: vec![invalid_node(); num_users], - users_writer: vec![invalid_node(); num_users], - users_used: vec![false; num_users], + rwu_table: RWUTable::new(num_live_nodes * num_vars), break_ln: NodeMap(), cont_ln: NodeMap(), } @@ -657,16 +753,13 @@ fn idx(&self, ln: LiveNode, var: Variable) -> usize { ln.get() * self.ir.num_vars + var.get() } - fn live_on_entry(&self, ln: LiveNode, var: Variable) - -> Option { + fn live_on_entry(&self, ln: LiveNode, var: Variable) -> Option { assert!(ln.is_valid()); - let reader = self.users_reader[self.idx(ln, var)]; - if reader.is_valid() {Some(self.ir.lnk(reader))} else {None} + let reader = self.rwu_table.get_reader(self.idx(ln, var)); + if reader.is_valid() { Some(self.ir.lnk(reader)) } else { None } } - /* - Is this variable live on entry to any of its successor nodes? - */ + // Is this variable live on entry to any of its successor nodes? fn live_on_exit(&self, ln: LiveNode, var: Variable) -> Option { let successor = self.successors[ln.get()]; @@ -675,14 +768,14 @@ fn live_on_exit(&self, ln: LiveNode, var: Variable) fn used_on_entry(&self, ln: LiveNode, var: Variable) -> bool { assert!(ln.is_valid()); - self.users_used[self.idx(ln, var)] + self.rwu_table.get_used(self.idx(ln, var)) } fn assigned_on_entry(&self, ln: LiveNode, var: Variable) -> Option { assert!(ln.is_valid()); - let writer = self.users_writer[self.idx(ln, var)]; - if writer.is_valid() {Some(self.ir.lnk(writer))} else {None} + let writer = self.rwu_table.get_writer(self.idx(ln, var)); + if writer.is_valid() { Some(self.ir.lnk(writer)) } else { None } } fn assigned_on_exit(&self, ln: LiveNode, var: Variable) @@ -725,9 +818,9 @@ fn ln_str(&self, ln: LiveNode) -> String { { let wr = &mut wr as &mut dyn Write; write!(wr, "[ln({:?}) of kind {:?} reads", ln.get(), self.ir.lnk(ln)); - self.write_vars(wr, ln, |idx| self.users_reader[idx]); + self.write_vars(wr, ln, |idx| self.rwu_table.get_reader(idx)); write!(wr, " writes"); - self.write_vars(wr, ln, |idx| self.users_writer[idx]); + self.write_vars(wr, ln, |idx| self.rwu_table.get_writer(idx)); write!(wr, " precedes {:?}]", self.successors[ln.get()]); } String::from_utf8(wr).unwrap() @@ -736,16 +829,9 @@ fn ln_str(&self, ln: LiveNode) -> String { fn init_empty(&mut self, ln: LiveNode, succ_ln: LiveNode) { self.successors[ln.get()] = succ_ln; - // It is not necessary to initialize the - // values to empty because this is the value - // they have when they are created, and the sets - // only grow during iterations. - // - // self.indices(ln) { |idx| - // self.users_reader[idx] = invalid_node(); - // self.users_writer[idx] = invalid_node(); - // self.users_used[idx] = false; - // } + // It is not necessary to initialize the RWUs here because they are all + // set to INV_INV_FALSE when they are created, and the sets only grow + // during iterations. } fn init_from_succ(&mut self, ln: LiveNode, succ_ln: LiveNode) { @@ -753,9 +839,7 @@ fn init_from_succ(&mut self, ln: LiveNode, succ_ln: LiveNode) { self.successors[ln.get()] = succ_ln; self.indices2(ln, succ_ln, |this, idx, succ_idx| { - this.users_reader[idx] = this.users_reader[succ_idx]; - this.users_writer[idx] = this.users_writer[succ_idx]; - this.users_used[idx] = this.users_used[succ_idx]; + this.rwu_table.copy_packed(idx, succ_idx); }); debug!("init_from_succ(ln={}, succ={})", self.ln_str(ln), self.ln_str(succ_ln)); @@ -770,26 +854,31 @@ fn merge_from_succ(&mut self, let mut changed = false; self.indices2(ln, succ_ln, |this, idx, succ_idx| { - changed |= copy_if_invalid(this.users_reader[succ_idx], &mut this.users_reader[idx]); - changed |= copy_if_invalid(this.users_writer[succ_idx], &mut this.users_writer[idx]); - if this.users_used[succ_idx] && !this.users_used[idx] { - this.users_used[idx] = true; + let mut rwu = this.rwu_table.get(idx); + let succ_rwu = this.rwu_table.get(succ_idx); + if succ_rwu.reader.is_valid() && !rwu.reader.is_valid() { + rwu.reader = succ_rwu.reader; + changed = true + } + + if succ_rwu.writer.is_valid() && !rwu.writer.is_valid() { + rwu.writer = succ_rwu.writer; + changed = true + } + + if succ_rwu.used && !rwu.used { + rwu.used = true; changed = true; } + + if changed { + this.rwu_table.assign_unpacked(idx, rwu); + } }); debug!("merge_from_succ(ln={:?}, succ={}, first_merge={}, changed={})", ln, self.ln_str(succ_ln), first_merge, changed); return changed; - - fn copy_if_invalid(src: LiveNode, dst: &mut LiveNode) -> bool { - if src.is_valid() && !dst.is_valid() { - *dst = src; - true - } else { - false - } - } } // Indicates that a local variable was *defined*; we know that no @@ -797,8 +886,7 @@ fn copy_if_invalid(src: LiveNode, dst: &mut LiveNode) -> bool { // this) so we just clear out all the data. fn define(&mut self, writer: LiveNode, var: Variable) { let idx = self.idx(writer, var); - self.users_reader[idx] = invalid_node(); - self.users_writer[idx] = invalid_node(); + self.rwu_table.assign_inv_inv(idx); debug!("{:?} defines {:?} (idx={}): {}", writer, var, idx, self.ln_str(writer)); @@ -810,21 +898,24 @@ fn acc(&mut self, ln: LiveNode, var: Variable, acc: u32) { ln, acc, var, self.ln_str(ln)); let idx = self.idx(ln, var); + let mut rwu = self.rwu_table.get(idx); if (acc & ACC_WRITE) != 0 { - self.users_reader[idx] = invalid_node(); - self.users_writer[idx] = ln; + rwu.reader = invalid_node(); + rwu.writer = ln; } // Important: if we both read/write, must do read second // or else the write will override. if (acc & ACC_READ) != 0 { - self.users_reader[idx] = ln; + rwu.reader = ln; } if (acc & ACC_USE) != 0 { - self.users_used[idx] = true; + rwu.used = true; } + + self.rwu_table.assign_unpacked(idx, rwu); } // _______________________________________________________________________ -- 2.44.0