From: Nicholas Nethercote <nnethercote@mozilla.com>
Date: Tue, 14 Aug 2018 00:21:24 +0000 (+1000)
Subject: Speed up NLL with `HybridIdxSetBuf`.
X-Git-Url: https://git.lizzy.rs/?a=commitdiff_plain;h=5745597e6195fe0591737f242d02350001b6c590;p=rust.git

Speed up NLL with `HybridIdxSetBuf`.

`HybridIdxSetBuf` is a sparse-when-small but dense-when-large index set
that is very efficient for sets that (a) have few elements, (b) have
large `universe_size` values, and (c) are cleared frequently. Which
makes it perfect for the `gen_set` and `kill_set` sets used by the new
borrow checker.

This patch reduces the execution time of the five slowest NLL benchmarks
by 55%, 21%, 16%, 10% and 9%. It also reduces the max-rss of three
benchmarks by 53%, 33%, and 9%.
---

diff --git a/src/librustc_data_structures/indexed_set.rs b/src/librustc_data_structures/indexed_set.rs
index 2e95a45479c..ed2e9787800 100644
--- a/src/librustc_data_structures/indexed_set.rs
+++ b/src/librustc_data_structures/indexed_set.rs
@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+use array_vec::ArrayVec;
 use std::borrow::{Borrow, BorrowMut, ToOwned};
 use std::fmt;
 use std::iter;
@@ -25,6 +26,8 @@
 ///
 /// In other words, `T` is the type used to index into the bitvector
 /// this type uses to represent the set of object it holds.
+///
+/// The representation is dense, using one bit per possible element.
 #[derive(Eq, PartialEq)]
 pub struct IdxSetBuf<T: Idx> {
     _pd: PhantomData<fn(&T)>,
@@ -93,6 +96,8 @@ fn to_owned(&self) -> Self::Owned {
     }
 }
 
+const BITS_PER_WORD: usize = mem::size_of::<Word>() * 8;
+
 impl<T: Idx> fmt::Debug for IdxSetBuf<T> {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
         w.debug_list()
@@ -111,8 +116,7 @@ fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
 
 impl<T: Idx> IdxSetBuf<T> {
     fn new(init: Word, universe_size: usize) -> Self {
-        let bits_per_word = mem::size_of::<Word>() * 8;
-        let num_words = (universe_size + (bits_per_word - 1)) / bits_per_word;
+        let num_words = (universe_size + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
         IdxSetBuf {
             _pd: Default::default(),
             bits: vec![init; num_words],
@@ -163,6 +167,16 @@ pub fn to_owned(&self) -> IdxSetBuf<T> {
         }
     }
 
+    /// Duplicates as a hybrid set.
+    pub fn to_hybrid(&self) -> HybridIdxSetBuf<T> {
+        // This universe_size may be slightly larger than the one specified
+        // upon creation, due to rounding up to a whole word. That's ok.
+        let universe_size = self.bits.len() * BITS_PER_WORD;
+
+        // Note: we currently don't bother trying to make a Sparse set.
+        HybridIdxSetBuf::Dense(self.to_owned(), universe_size)
+    }
+
     /// Removes all elements
     pub fn clear(&mut self) {
         for b in &mut self.bits {
@@ -180,11 +194,9 @@ pub fn set_up_to(&mut self, universe_size: usize) {
 
     /// Clear all elements above `universe_size`.
     fn trim_to(&mut self, universe_size: usize) {
-        let word_bits = mem::size_of::<Word>() * 8;
-
         // `trim_block` is the first block where some bits have
         // to be cleared.
-        let trim_block = universe_size / word_bits;
+        let trim_block = universe_size / BITS_PER_WORD;
 
         // all the blocks above it have to be completely cleared.
         if trim_block < self.bits.len() {
@@ -192,9 +204,9 @@ fn trim_to(&mut self, universe_size: usize) {
                 *b = 0;
             }
 
-            // at that block, the `universe_size % word_bits` lsbs
+            // at that block, the `universe_size % BITS_PER_WORD` lsbs
             // should remain.
-            let remaining_bits = universe_size % word_bits;
+            let remaining_bits = universe_size % BITS_PER_WORD;
             let mask = (1<<remaining_bits)-1;
             self.bits[trim_block] &= mask;
         }
@@ -245,12 +257,46 @@ pub fn union(&mut self, other: &IdxSet<T>) -> bool {
         bitwise(self.words_mut(), other.words(), &Union)
     }
 
+    /// Like `union()`, but takes a `SparseIdxSetBuf` argument.
+    fn union_sparse(&mut self, other: &SparseIdxSetBuf<T>) -> bool {
+        let mut changed = false;
+        for elem in other.iter() {
+            changed |= self.add(&elem);
+        }
+        changed
+    }
+
+    /// Like `union()`, but takes a `HybridIdxSetBuf` argument.
+    pub fn union_hybrid(&mut self, other: &HybridIdxSetBuf<T>) -> bool {
+        match other {
+            HybridIdxSetBuf::Sparse(sparse, _) => self.union_sparse(sparse),
+            HybridIdxSetBuf::Dense(dense, _) => self.union(dense),
+        }
+    }
+
     /// Set `self = self - other` and return true if `self` changed.
     /// (i.e., if any bits were removed).
     pub fn subtract(&mut self, other: &IdxSet<T>) -> bool {
         bitwise(self.words_mut(), other.words(), &Subtract)
     }
 
+    /// Like `subtract()`, but takes a `SparseIdxSetBuf` argument.
+    fn subtract_sparse(&mut self, other: &SparseIdxSetBuf<T>) -> bool {
+        let mut changed = false;
+        for elem in other.iter() {
+            changed |= self.remove(&elem);
+        }
+        changed
+    }
+
+    /// Like `subtract()`, but takes a `HybridIdxSetBuf` argument.
+    pub fn subtract_hybrid(&mut self, other: &HybridIdxSetBuf<T>) -> bool {
+        match other {
+            HybridIdxSetBuf::Sparse(sparse, _) => self.subtract_sparse(sparse),
+            HybridIdxSetBuf::Dense(dense, _) => self.subtract(dense),
+        }
+    }
+
     /// Set `self = self & other` and return true if `self` changed.
     /// (i.e., if any bits were removed).
     pub fn intersect(&mut self, other: &IdxSet<T>) -> bool {
@@ -276,11 +322,10 @@ impl<'a, T: Idx> Iterator for Iter<'a, T> {
     type Item = T;
 
     fn next(&mut self) -> Option<T> {
-        let word_bits = mem::size_of::<Word>() * 8;
         loop {
             if let Some((ref mut word, offset)) = self.cur {
                 let bit_pos = word.trailing_zeros() as usize;
-                if bit_pos != word_bits {
+                if bit_pos != BITS_PER_WORD {
                     let bit = 1 << bit_pos;
                     *word ^= bit;
                     return Some(T::new(bit_pos + offset))
@@ -288,7 +333,189 @@ fn next(&mut self) -> Option<T> {
             }
 
             let (i, word) = self.iter.next()?;
-            self.cur = Some((*word, word_bits * i));
+            self.cur = Some((*word, BITS_PER_WORD * i));
+        }
+    }
+}
+
+const SPARSE_MAX: usize = 8;
+
+/// A sparse index set with a maximum of SPARSE_MAX elements. Used by
+/// HybridIdxSetBuf; do not use directly.
+///
+/// The elements are stored as an unsorted vector with no duplicates.
+#[derive(Clone, Debug)]
+pub struct SparseIdxSetBuf<T: Idx>(ArrayVec<[T; SPARSE_MAX]>);
+
+impl<T: Idx> SparseIdxSetBuf<T> {
+    fn new() -> Self {
+        SparseIdxSetBuf(ArrayVec::new())
+    }
+
+    fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    fn contains(&self, elem: &T) -> bool {
+        self.0.contains(elem)
+    }
+
+    fn add(&mut self, elem: &T) -> bool {
+        // Ensure there are no duplicates.
+        if self.0.contains(elem) {
+            false
+        } else {
+            self.0.push(*elem);
+            true
+        }
+    }
+
+    fn remove(&mut self, elem: &T) -> bool {
+        if let Some(i) = self.0.iter().position(|e| e == elem) {
+            // Swap the found element to the end, then pop it.
+            let len = self.0.len();
+            self.0.swap(i, len - 1);
+            self.0.pop();
+            true
+        } else {
+            false
+        }
+    }
+
+    fn to_dense(&self, universe_size: usize) -> IdxSetBuf<T> {
+        let mut dense = IdxSetBuf::new_empty(universe_size);
+        for elem in self.0.iter() {
+            dense.add(elem);
+        }
+        dense
+    }
+
+    fn iter(&self) -> SparseIter<T> {
+        SparseIter {
+            iter: self.0.iter(),
+        }
+    }
+}
+
+pub struct SparseIter<'a, T: Idx> {
+    iter: slice::Iter<'a, T>,
+}
+
+impl<'a, T: Idx> Iterator for SparseIter<'a, T> {
+    type Item = T;
+
+    fn next(&mut self) -> Option<T> {
+        self.iter.next().map(|e| *e)
+    }
+}
+
+/// Like IdxSetBuf, but with a hybrid representation: sparse when there are few
+/// elements in the set, but dense when there are many. It's especially
+/// efficient for sets that typically have a small number of elements, but a
+/// large `universe_size`, and are cleared frequently.
+#[derive(Clone, Debug)]
+pub enum HybridIdxSetBuf<T: Idx> {
+    Sparse(SparseIdxSetBuf<T>, usize),
+    Dense(IdxSetBuf<T>, usize),
+}
+
+impl<T: Idx> HybridIdxSetBuf<T> {
+    pub fn new_empty(universe_size: usize) -> Self {
+        HybridIdxSetBuf::Sparse(SparseIdxSetBuf::new(), universe_size)
+    }
+
+    fn universe_size(&mut self) -> usize {
+        match *self {
+            HybridIdxSetBuf::Sparse(_, size) => size,
+            HybridIdxSetBuf::Dense(_, size) => size,
+        }
+    }
+
+    pub fn clear(&mut self) {
+        let universe_size = self.universe_size();
+        *self = HybridIdxSetBuf::new_empty(universe_size);
+    }
+
+    /// Returns true iff set `self` contains `elem`.
+    pub fn contains(&self, elem: &T) -> bool {
+        match self {
+            HybridIdxSetBuf::Sparse(sparse, _) => sparse.contains(elem),
+            HybridIdxSetBuf::Dense(dense, _) => dense.contains(elem),
+        }
+    }
+
+    /// Adds `elem` to the set `self`.
+    pub fn add(&mut self, elem: &T) -> bool {
+        match self {
+            HybridIdxSetBuf::Sparse(sparse, _) if sparse.len() < SPARSE_MAX => {
+                // The set is sparse and has space for `elem`.
+                sparse.add(elem)
+            }
+            HybridIdxSetBuf::Sparse(sparse, _) if sparse.contains(elem) => {
+                // The set is sparse and does not have space for `elem`, but
+                // that doesn't matter because `elem` is already present.
+                false
+            }
+            HybridIdxSetBuf::Sparse(_, _) => {
+                // The set is sparse and full. Convert to a dense set.
+                //
+                // FIXME: This code is awful, but I can't work out how else to
+                //        appease the borrow checker.
+                let dummy = HybridIdxSetBuf::Sparse(SparseIdxSetBuf::new(), 0);
+                match mem::replace(self, dummy) {
+                    HybridIdxSetBuf::Sparse(sparse, universe_size) => {
+                        let mut dense = sparse.to_dense(universe_size);
+                        let changed = dense.add(elem);
+                        assert!(changed);
+                        mem::replace(self, HybridIdxSetBuf::Dense(dense, universe_size));
+                        changed
+                    }
+                    _ => panic!("impossible"),
+                }
+            }
+
+            HybridIdxSetBuf::Dense(dense, _) => dense.add(elem),
+        }
+    }
+
+    /// Removes `elem` from the set `self`.
+    pub fn remove(&mut self, elem: &T) -> bool {
+        // Note: we currently don't bother going from Dense back to Sparse.
+        match self {
+            HybridIdxSetBuf::Sparse(sparse, _) => sparse.remove(elem),
+            HybridIdxSetBuf::Dense(dense, _) => dense.remove(elem),
+        }
+    }
+
+    /// Converts to a dense set, consuming itself in the process.
+    pub fn to_dense(self) -> IdxSetBuf<T> {
+        match self {
+            HybridIdxSetBuf::Sparse(sparse, universe_size) => sparse.to_dense(universe_size),
+            HybridIdxSetBuf::Dense(dense, _) => dense,
+        }
+    }
+
+    /// Iteration order is unspecified.
+    pub fn iter(&self) -> HybridIter<T> {
+        match self {
+            HybridIdxSetBuf::Sparse(sparse, _) => HybridIter::Sparse(sparse.iter()),
+            HybridIdxSetBuf::Dense(dense, _) => HybridIter::Dense(dense.iter()),
+        }
+    }
+}
+
+pub enum HybridIter<'a, T: Idx> {
+    Sparse(SparseIter<'a, T>),
+    Dense(Iter<'a, T>),
+}
+
+impl<'a, T: Idx> Iterator for HybridIter<'a, T> {
+    type Item = T;
+
+    fn next(&mut self) -> Option<T> {
+        match self {
+            HybridIter::Sparse(sparse) => sparse.next(),
+            HybridIter::Dense(dense) => dense.next(),
         }
     }
 }
diff --git a/src/librustc_mir/dataflow/at_location.rs b/src/librustc_mir/dataflow/at_location.rs
index 05453bd58c4..d2a8a9dcf4b 100644
--- a/src/librustc_mir/dataflow/at_location.rs
+++ b/src/librustc_mir/dataflow/at_location.rs
@@ -12,7 +12,7 @@
 //! locations.
 
 use rustc::mir::{BasicBlock, Location};
-use rustc_data_structures::indexed_set::{IdxSetBuf, Iter};
+use rustc_data_structures::indexed_set::{HybridIdxSetBuf, IdxSetBuf, Iter};
 use rustc_data_structures::indexed_vec::Idx;
 
 use dataflow::{BitDenotation, BlockSets, DataflowResults};
@@ -68,8 +68,8 @@ pub struct FlowAtLocation<BD>
 {
     base_results: DataflowResults<BD>,
     curr_state: IdxSetBuf<BD::Idx>,
-    stmt_gen: IdxSetBuf<BD::Idx>,
-    stmt_kill: IdxSetBuf<BD::Idx>,
+    stmt_gen: HybridIdxSetBuf<BD::Idx>,
+    stmt_kill: HybridIdxSetBuf<BD::Idx>,
 }
 
 impl<BD> FlowAtLocation<BD>
@@ -97,8 +97,8 @@ pub fn each_gen_bit<F>(&self, f: F)
     pub fn new(results: DataflowResults<BD>) -> Self {
         let bits_per_block = results.sets().bits_per_block();
         let curr_state = IdxSetBuf::new_empty(bits_per_block);
-        let stmt_gen = IdxSetBuf::new_empty(bits_per_block);
-        let stmt_kill = IdxSetBuf::new_empty(bits_per_block);
+        let stmt_gen = HybridIdxSetBuf::new_empty(bits_per_block);
+        let stmt_kill = HybridIdxSetBuf::new_empty(bits_per_block);
         FlowAtLocation {
             base_results: results,
             curr_state: curr_state,
@@ -129,8 +129,8 @@ pub fn with_iter_outgoing<F>(&self, f: F)
         F: FnOnce(Iter<BD::Idx>),
     {
         let mut curr_state = self.curr_state.clone();
-        curr_state.union(&self.stmt_gen);
-        curr_state.subtract(&self.stmt_kill);
+        curr_state.union_hybrid(&self.stmt_gen);
+        curr_state.subtract_hybrid(&self.stmt_kill);
         f(curr_state.iter());
     }
 }
@@ -193,8 +193,8 @@ fn reconstruct_terminator_effect(&mut self, loc: Location) {
     }
 
     fn apply_local_effect(&mut self, _loc: Location) {
-        self.curr_state.union(&self.stmt_gen);
-        self.curr_state.subtract(&self.stmt_kill);
+        self.curr_state.union_hybrid(&self.stmt_gen);
+        self.curr_state.subtract_hybrid(&self.stmt_kill);
     }
 }
 
diff --git a/src/librustc_mir/dataflow/graphviz.rs b/src/librustc_mir/dataflow/graphviz.rs
index 7475b4d82f4..97d54998771 100644
--- a/src/librustc_mir/dataflow/graphviz.rs
+++ b/src/librustc_mir/dataflow/graphviz.rs
@@ -168,13 +168,13 @@ fn node_label_verbose_row<W: io::Write>(&self,
         let i = n.index();
 
         macro_rules! dump_set_for {
-            ($set:ident) => {
+            ($set:ident, $interpret:ident) => {
                 write!(w, "<td>")?;
 
                 let flow = self.mbcx.flow_state();
-                let entry_interp = flow.interpret_set(&flow.operator,
-                                                      flow.sets.$set(i),
-                                                      &self.render_idx);
+                let entry_interp = flow.$interpret(&flow.operator,
+                                                   flow.sets.$set(i),
+                                                   &self.render_idx);
                 for e in &entry_interp {
                     write!(w, "{:?}<br/>", e)?;
                 }
@@ -184,7 +184,7 @@ macro_rules! dump_set_for {
 
         write!(w, "<tr>")?;
         // Entry
-        dump_set_for!(on_entry_set_for);
+        dump_set_for!(on_entry_set_for, interpret_set);
 
         // MIR statements
         write!(w, "<td>")?;
@@ -198,10 +198,10 @@ macro_rules! dump_set_for {
         write!(w, "</td>")?;
 
         // Gen
-        dump_set_for!(gen_set_for);
+        dump_set_for!(gen_set_for, interpret_hybrid_set);
 
         // Kill
-        dump_set_for!(kill_set_for);
+        dump_set_for!(kill_set_for, interpret_hybrid_set);
 
         write!(w, "</tr>")?;
 
@@ -217,19 +217,14 @@ fn node_label_final_row<W: io::Write>(&self,
                                           -> io::Result<()> {
         let i = n.index();
 
-        macro_rules! dump_set_for {
-            ($set:ident) => {
-                let flow = self.mbcx.flow_state();
-                let bits_per_block = flow.sets.bits_per_block();
-                let set = flow.sets.$set(i);
-                write!(w, "<td>{:?}</td>",
-                       dot::escape_html(&bits_to_string(set.words(), bits_per_block)))?;
-            }
-        }
+        let flow = self.mbcx.flow_state();
+        let bits_per_block = flow.sets.bits_per_block();
 
         write!(w, "<tr>")?;
+
         // Entry
-        dump_set_for!(on_entry_set_for);
+        let set = flow.sets.on_entry_set_for(i);
+        write!(w, "<td>{:?}</td>", dot::escape_html(&bits_to_string(set.words(), bits_per_block)))?;
 
         // Terminator
         write!(w, "<td>")?;
@@ -242,10 +237,12 @@ macro_rules! dump_set_for {
         write!(w, "</td>")?;
 
         // Gen
-        dump_set_for!(gen_set_for);
+        let set = flow.sets.gen_set_for(i);
+        write!(w, "<td>{:?}</td>", dot::escape_html(&format!("{:?}", set)))?;
 
         // Kill
-        dump_set_for!(kill_set_for);
+        let set = flow.sets.kill_set_for(i);
+        write!(w, "<td>{:?}</td>", dot::escape_html(&format!("{:?}", set)))?;
 
         write!(w, "</tr>")?;
 
diff --git a/src/librustc_mir/dataflow/mod.rs b/src/librustc_mir/dataflow/mod.rs
index 4227f0bcd36..56c4dac19e5 100644
--- a/src/librustc_mir/dataflow/mod.rs
+++ b/src/librustc_mir/dataflow/mod.rs
@@ -10,9 +10,9 @@
 
 use syntax::ast::{self, MetaItem};
 
-use rustc_data_structures::indexed_set::{IdxSet, IdxSetBuf};
-use rustc_data_structures::indexed_vec::Idx;
 use rustc_data_structures::bitslice::{bitwise, BitwiseOperator, Word};
+use rustc_data_structures::indexed_set::{HybridIdxSetBuf, IdxSet, IdxSetBuf};
+use rustc_data_structures::indexed_vec::Idx;
 use rustc_data_structures::work_queue::WorkQueue;
 
 use rustc::ty::{self, TyCtxt};
@@ -188,7 +188,6 @@ fn propagate(&mut self) {
             builder: self,
         };
         propcx.walk_cfg(&mut temp);
-
     }
 
     fn build_sets(&mut self) {
@@ -243,8 +242,8 @@ fn walk_cfg(&mut self, in_out: &mut IdxSet<BD::Idx>) {
                 let sets = self.builder.flow_state.sets.for_block(bb.index());
                 debug_assert!(in_out.words().len() == sets.on_entry.words().len());
                 in_out.overwrite(sets.on_entry);
-                in_out.union(sets.gen_set);
-                in_out.subtract(sets.kill_set);
+                in_out.union_hybrid(sets.gen_set);
+                in_out.subtract_hybrid(sets.kill_set);
             }
             self.builder.propagate_bits_into_graph_successors_of(
                 in_out, (bb, bb_data), &mut dirty_queue);
@@ -289,15 +288,11 @@ fn post_dataflow_instrumentation<P>(&self, p: P) -> io::Result<()>
 }
 
 /// Maps each block to a set of bits
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub(crate) struct Bits<E:Idx> {
     bits: IdxSetBuf<E>,
 }
 
-impl<E:Idx> Clone for Bits<E> {
-    fn clone(&self) -> Self { Bits { bits: self.bits.clone() } }
-}
-
 impl<E:Idx> Bits<E> {
     fn new(bits: IdxSetBuf<E>) -> Self {
         Bits { bits: bits }
@@ -372,13 +367,15 @@ pub fn state_for_location<'tcx, T: BitDenotation>(loc: Location,
                                                   result: &DataflowResults<T>,
                                                   mir: &Mir<'tcx>)
     -> IdxSetBuf<T::Idx> {
-    let mut entry = result.sets().on_entry_set_for(loc.block.index()).to_owned();
+    let mut on_entry = result.sets().on_entry_set_for(loc.block.index()).to_owned();
+    let mut kill_set = on_entry.to_hybrid();
+    let mut gen_set = kill_set.clone();
 
     {
         let mut sets = BlockSets {
-            on_entry: &mut entry.clone(),
-            kill_set: &mut entry.clone(),
-            gen_set: &mut entry,
+            on_entry: &mut on_entry,
+            kill_set: &mut kill_set,
+            gen_set: &mut gen_set,
         };
 
         for stmt in 0..loc.statement_index {
@@ -396,7 +393,7 @@ pub fn state_for_location<'tcx, T: BitDenotation>(loc: Location,
         }
     }
 
-    entry
+    gen_set.to_dense()
 }
 
 pub struct DataflowAnalysis<'a, 'tcx: 'a, O> where O: BitDenotation
@@ -443,12 +440,22 @@ pub struct DataflowState<O: BitDenotation>
 impl<O: BitDenotation> DataflowState<O> {
     pub(crate) fn interpret_set<'c, P>(&self,
                                        o: &'c O,
-                                       words: &IdxSet<O::Idx>,
+                                       set: &IdxSet<O::Idx>,
                                        render_idx: &P)
                                        -> Vec<DebugFormatted>
         where P: Fn(&O, O::Idx) -> DebugFormatted
     {
-        words.iter().map(|i| render_idx(o, i)).collect()
+        set.iter().map(|i| render_idx(o, i)).collect()
+    }
+
+    pub(crate) fn interpret_hybrid_set<'c, P>(&self,
+                                              o: &'c O,
+                                              set: &HybridIdxSetBuf<O::Idx>,
+                                              render_idx: &P)
+                                              -> Vec<DebugFormatted>
+        where P: Fn(&O, O::Idx) -> DebugFormatted
+    {
+        set.iter().map(|i| render_idx(o, i)).collect()
     }
 }
 
@@ -461,18 +468,18 @@ pub struct AllSets<E: Idx> {
     /// equal to bits_per_block / (mem::size_of::<Word> * 8), rounded up.
     words_per_block: usize,
 
+    /// For each block, bits valid on entry to the block.
+    on_entry_sets: Bits<E>,
+
     /// For each block, bits generated by executing the statements in
     /// the block. (For comparison, the Terminator for each block is
     /// handled in a flow-specific manner during propagation.)
-    gen_sets: Bits<E>,
+    gen_sets: Vec<HybridIdxSetBuf<E>>,
 
     /// For each block, bits killed by executing the statements in the
     /// block. (For comparison, the Terminator for each block is
     /// handled in a flow-specific manner during propagation.)
-    kill_sets: Bits<E>,
-
-    /// For each block, bits valid on entry to the block.
-    on_entry_sets: Bits<E>,
+    kill_sets: Vec<HybridIdxSetBuf<E>>,
 }
 
 /// Triple of sets associated with a given block.
@@ -494,11 +501,13 @@ pub struct BlockSets<'a, E: Idx> {
     /// Dataflow state immediately before control flow enters the given block.
     pub(crate) on_entry: &'a mut IdxSet<E>,
 
-    /// Bits that are set to 1 by the time we exit the given block.
-    pub(crate) gen_set: &'a mut IdxSet<E>,
+    /// Bits that are set to 1 by the time we exit the given block. Hybrid
+    /// because it usually contains only 0 or 1 elements.
+    pub(crate) gen_set: &'a mut HybridIdxSetBuf<E>,
 
-    /// Bits that are set to 0 by the time we exit the given block.
-    pub(crate) kill_set: &'a mut IdxSet<E>,
+    /// Bits that are set to 0 by the time we exit the given block. Hybrid
+    /// because it usually contains only 0 or 1 elements.
+    pub(crate) kill_set: &'a mut HybridIdxSetBuf<E>,
 }
 
 impl<'a, E:Idx> BlockSets<'a, E> {
@@ -542,8 +551,8 @@ fn kill_all<I>(&mut self, i: I)
     }
 
     fn apply_local_effect(&mut self) {
-        self.on_entry.union(&self.gen_set);
-        self.on_entry.subtract(&self.kill_set);
+        self.on_entry.union_hybrid(&self.gen_set);
+        self.on_entry.subtract_hybrid(&self.kill_set);
     }
 }
 
@@ -554,24 +563,21 @@ pub fn for_block(&mut self, block_idx: usize) -> BlockSets<E> {
         let range = E::new(offset)..E::new(offset + self.words_per_block);
         BlockSets {
             on_entry: self.on_entry_sets.bits.range_mut(&range),
-            gen_set: self.gen_sets.bits.range_mut(&range),
-            kill_set: self.kill_sets.bits.range_mut(&range),
+            gen_set: &mut self.gen_sets[block_idx],
+            kill_set: &mut self.kill_sets[block_idx],
         }
     }
 
-    fn lookup_set_for<'a>(&self, sets: &'a Bits<E>, block_idx: usize) -> &'a IdxSet<E> {
+    pub fn on_entry_set_for(&self, block_idx: usize) -> &IdxSet<E> {
         let offset = self.words_per_block * block_idx;
         let range = E::new(offset)..E::new(offset + self.words_per_block);
-        sets.bits.range(&range)
+        self.on_entry_sets.bits.range(&range)
     }
-    pub fn gen_set_for(&self, block_idx: usize) -> &IdxSet<E> {
-        self.lookup_set_for(&self.gen_sets, block_idx)
+    pub fn gen_set_for(&self, block_idx: usize) -> &HybridIdxSetBuf<E> {
+        &self.gen_sets[block_idx]
     }
-    pub fn kill_set_for(&self, block_idx: usize) -> &IdxSet<E> {
-        self.lookup_set_for(&self.kill_sets, block_idx)
-    }
-    pub fn on_entry_set_for(&self, block_idx: usize) -> &IdxSet<E> {
-        self.lookup_set_for(&self.on_entry_sets, block_idx)
+    pub fn kill_set_for(&self, block_idx: usize) -> &HybridIdxSetBuf<E> {
+        &self.kill_sets[block_idx]
     }
 }
 
@@ -731,12 +737,12 @@ pub fn new(mir: &'a Mir<'tcx>,
         let num_blocks = mir.basic_blocks().len();
         let num_overall = num_blocks * bits_per_block_rounded_up;
 
-        let zeroes = Bits::new(IdxSetBuf::new_empty(num_overall));
         let on_entry = Bits::new(if D::bottom_value() {
             IdxSetBuf::new_filled(num_overall)
         } else {
             IdxSetBuf::new_empty(num_overall)
         });
+        let empties = vec![HybridIdxSetBuf::new_empty(bits_per_block); num_blocks];
 
         DataflowAnalysis {
             mir,
@@ -745,9 +751,9 @@ pub fn new(mir: &'a Mir<'tcx>,
                 sets: AllSets {
                     bits_per_block,
                     words_per_block,
-                    gen_sets: zeroes.clone(),
-                    kill_sets: zeroes,
                     on_entry_sets: on_entry,
+                    gen_sets: empties.clone(),
+                    kill_sets: empties,
                 },
                 operator: denotation,
             }
diff --git a/src/librustc_mir/transform/rustc_peek.rs b/src/librustc_mir/transform/rustc_peek.rs
index da149f42064..776d7888459 100644
--- a/src/librustc_mir/transform/rustc_peek.rs
+++ b/src/librustc_mir/transform/rustc_peek.rs
@@ -138,9 +138,9 @@ fn each_block<'a, 'tcx, O>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         }
     };
 
-    let mut entry = results.0.sets.on_entry_set_for(bb.index()).to_owned();
-    let mut gen = results.0.sets.gen_set_for(bb.index()).to_owned();
-    let mut kill = results.0.sets.kill_set_for(bb.index()).to_owned();
+    let mut on_entry = results.0.sets.on_entry_set_for(bb.index()).to_owned();
+    let mut gen_set = results.0.sets.gen_set_for(bb.index()).clone();
+    let mut kill_set = results.0.sets.kill_set_for(bb.index()).clone();
 
     // Emulate effect of all statements in the block up to (but not
     // including) the borrow within `peek_arg_place`. Do *not* include
@@ -148,9 +148,9 @@ fn each_block<'a, 'tcx, O>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
     // of the argument at time immediate preceding Call to
     // `rustc_peek`).
 
-    let mut sets = dataflow::BlockSets { on_entry: &mut entry,
-                                      gen_set: &mut gen,
-                                      kill_set: &mut kill };
+    let mut sets = dataflow::BlockSets { on_entry: &mut on_entry,
+                                         gen_set: &mut gen_set,
+                                         kill_set: &mut kill_set };
 
     for (j, stmt) in statements.iter().enumerate() {
         debug!("rustc_peek: ({:?},{}) {:?}", bb, j, stmt);
@@ -203,14 +203,14 @@ fn each_block<'a, 'tcx, O>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         debug!("rustc_peek: computing effect on place: {:?} ({:?}) in stmt: {:?}",
                place, lhs_mpi, stmt);
         // reset GEN and KILL sets before emulating their effect.
-        for e in sets.gen_set.words_mut() { *e = 0; }
-        for e in sets.kill_set.words_mut() { *e = 0; }
+        sets.gen_set.clear();
+        sets.kill_set.clear();
         results.0.operator.before_statement_effect(
             &mut sets, Location { block: bb, statement_index: j });
         results.0.operator.statement_effect(
             &mut sets, Location { block: bb, statement_index: j });
-        sets.on_entry.union(sets.gen_set);
-        sets.on_entry.subtract(sets.kill_set);
+        sets.on_entry.union_hybrid(sets.gen_set);
+        sets.on_entry.subtract_hybrid(sets.kill_set);
     }
 
     results.0.operator.before_terminator_effect(