Work products are deleted if any of their inputs are dirty.
// We need nested scopes here, because the intermediate results can keep
// large chunks of memory alive and we want to free them as soon as
// possible to keep the peak memory usage low
- let (outputs, trans) = {
+ let (outputs, trans, id) = {
let krate = match phase_1_parse_input(sess, cfg, input) {
Ok(krate) => krate,
Err(mut parse_error) => {
// Discard interned strings as they are no longer required.
token::clear_ident_interner();
- Ok((outputs, trans))
+ Ok((outputs, trans, id.clone()))
})??
};
- let phase5_result = phase_5_run_llvm_passes(sess, &trans, &outputs);
+ let phase5_result = phase_5_run_llvm_passes(sess, &id, &trans, &outputs);
controller_entry_point!(after_llvm,
sess,
/// Run LLVM itself, producing a bitcode file, assembly file or object file
/// as a side effect.
pub fn phase_5_run_llvm_passes(sess: &Session,
+ crate_name: &str,
trans: &trans::CrateTranslation,
outputs: &OutputFilenames) -> CompileResult {
if sess.opts.cg.no_integrated_as {
|| write::run_passes(sess, trans, &sess.opts.output_types, outputs));
}
+ time(sess.time_passes(),
+ "serialize work products",
+ move || rustc_incremental::save_work_products(sess, crate_name));
+
if sess.err_count() > 0 {
Err(sess.err_count())
} else {
html_root_url = "https://doc.rust-lang.org/nightly/")]
#![cfg_attr(not(stage0), deny(warnings))]
+#![feature(question_mark)]
#![feature(rustc_private)]
#![feature(staged_api)]
pub use calculate_svh::SvhCalculate;
pub use persist::load_dep_graph;
pub use persist::save_dep_graph;
+pub use persist::save_work_products;
//! The data that we will serialize and deserialize.
-use rustc::dep_graph::DepNode;
+use rustc::dep_graph::{DepNode, WorkProduct, WorkProductId};
use rustc::hir::def_id::DefIndex;
+use std::sync::Arc;
use super::directory::DefPathIndex;
pub hash: u64,
}
+#[derive(Debug, RustcEncodable, RustcDecodable)]
+pub struct SerializedWorkProduct {
+ /// node that produced the work-product
+ pub id: Arc<WorkProductId>,
+
+ /// work-product data itself
+ pub work_product: WorkProduct,
+}
+
/// Data for use when downstream crates get recompiled.
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct SerializedMetadataHashes {
use rbml::opaque::Decoder;
use rustc::dep_graph::DepNode;
use rustc::hir::def_id::DefId;
+use rustc::session::Session;
use rustc::ty::TyCtxt;
use rustc_data_structures::fnv::FnvHashSet;
use rustc_serialize::Decodable as RustcDecodable;
use std::io::Read;
-use std::fs::File;
-use std::path::Path;
+use std::fs::{self, File};
+use std::path::{Path};
use super::data::*;
use super::directory::*;
/// actually it doesn't matter all that much.) See `README.md` for
/// more general overview.
pub fn load_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
+ if tcx.sess.opts.incremental.is_none() {
+ return;
+ }
+
let _ignore = tcx.dep_graph.in_ignore();
+ load_dep_graph_if_exists(tcx);
+ dirty_clean::check_dirty_clean_annotations(tcx);
+}
- if let Some(dep_graph) = dep_graph_path(tcx) {
- // FIXME(#32754) lock file?
- load_dep_graph_if_exists(tcx, &dep_graph);
- dirty_clean::check_dirty_clean_annotations(tcx);
+fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
+ let dep_graph_path = dep_graph_path(tcx).unwrap();
+ let dep_graph_data = match load_data(tcx.sess, &dep_graph_path) {
+ Some(p) => p,
+ None => return // no file
+ };
+
+ let work_products_path = tcx_work_products_path(tcx).unwrap();
+ let work_products_data = match load_data(tcx.sess, &work_products_path) {
+ Some(p) => p,
+ None => return // no file
+ };
+
+ match decode_dep_graph(tcx, &dep_graph_data, &work_products_data) {
+ Ok(()) => return,
+ Err(err) => bug!("decoding error in dep-graph from `{}` and `{}`: {}",
+ dep_graph_path.display(),
+ work_products_path.display(),
+ err),
}
}
-pub fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, path: &Path) {
+fn load_data(sess: &Session, path: &Path) -> Option<Vec<u8>> {
if !path.exists() {
- return;
+ return None;
}
let mut data = vec![];
File::open(path)
.and_then(|mut file| file.read_to_end(&mut data))
{
- Ok(_) => { }
+ Ok(_) => {
+ Some(data)
+ }
Err(err) => {
- tcx.sess.err(
+ sess.err(
&format!("could not load dep-graph from `{}`: {}",
path.display(), err));
- return;
+ None
}
}
- match decode_dep_graph(tcx, &data) {
- Ok(dirty) => dirty,
- Err(err) => {
- bug!("decoding error in dep-graph from `{}`: {}", path.display(), err);
- }
- }
}
+/// Decode the dep graph and load the edges/nodes that are still clean
+/// into `tcx.dep_graph`. On success, returns a hashset containing all
+/// the paths of work-products from clean nodes (any work-products not
+/// in this set can be deleted).
pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
- data: &[u8])
+ dep_graph_data: &[u8],
+ work_products_data: &[u8])
-> Result<(), Error>
{
// Deserialize the directory and dep-graph.
- let mut decoder = Decoder::new(data, 0);
- let directory = try!(DefIdDirectory::decode(&mut decoder));
- let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut decoder));
+ let mut dep_graph_decoder = Decoder::new(dep_graph_data, 0);
+ let directory = try!(DefIdDirectory::decode(&mut dep_graph_decoder));
+ let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut dep_graph_decoder));
debug!("decode_dep_graph: directory = {:#?}", directory);
debug!("decode_dep_graph: serialized_dep_graph = {:#?}", serialized_dep_graph);
// Add nodes and edges that are not dirty into our main graph.
let dep_graph = tcx.dep_graph.clone();
for (source, target) in clean_edges.into_iter().chain(clean_nodes) {
- let _task = dep_graph.in_task(target.clone());
- dep_graph.read(source.clone());
-
debug!("decode_dep_graph: clean edge: {:?} -> {:?}", source, target);
+
+ let _task = dep_graph.in_task(target);
+ dep_graph.read(source);
}
+ // Add in work-products that are still clean, and delete those that are
+ // dirty.
+ let mut work_product_decoder = Decoder::new(work_products_data, 0);
+ let work_products = try!(<Vec<SerializedWorkProduct>>::decode(&mut work_product_decoder));
+ reconcile_work_products(tcx, work_products, &dirty_nodes);
+
Ok(())
}
match hash.node.map_def(|&i| retraced.def_id(i)) {
Some(dep_node) => {
let current_hash = hcx.hash(&dep_node).unwrap();
- debug!("initial_dirty_nodes: hash of {:?} is {:?}, was {:?}",
- dep_node, current_hash, hash.hash);
if current_hash != hash.hash {
+ debug!("initial_dirty_nodes: {:?} is dirty as hash is {:?}, was {:?}",
+ dep_node, current_hash, hash.hash);
dirty_nodes.insert(dep_node);
}
}
clean_edges.push((source, target))
} else {
// source removed, target must be dirty
+ debug!("compute_clean_edges: {:?} dirty because {:?} no longer exists",
+ target, serialized_source);
dirty_nodes.insert(target);
}
} else {
clean_edges
}
+
+/// Go through the list of work-products produced in the previous run.
+/// Delete any whose nodes have been found to be dirty or which are
+/// otherwise no longer applicable.
+fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
+ work_products: Vec<SerializedWorkProduct>,
+ dirty_nodes: &DirtyNodes) {
+ debug!("reconcile_work_products({:?})", work_products);
+ for swp in work_products {
+ let dep_node = DepNode::WorkProduct(swp.id.clone());
+ if dirty_nodes.contains(&dep_node) {
+ debug!("reconcile_work_products: dep-node for {:?} is dirty", swp);
+ delete_dirty_work_product(tcx, swp);
+ } else {
+ let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
+ if path.exists() {
+ tcx.dep_graph.insert_previous_work_product(&swp.id, swp.work_product);
+ } else {
+ debug!("reconcile_work_products: file for {:?} does not exist", swp);
+ }
+ }
+ }
+}
+
+fn delete_dirty_work_product(tcx: TyCtxt,
+ swp: SerializedWorkProduct) {
+ debug!("delete_dirty_work_product({:?})", swp);
+ let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
+ match fs::remove_file(&path) {
+ Ok(()) => { }
+ Err(err) => {
+ tcx.sess.warn(
+ &format!("file-system error deleting outdated file `{}`: {}",
+ path.display(), err));
+ }
+ }
+}
pub use self::load::load_dep_graph;
pub use self::save::save_dep_graph;
+pub use self::save::save_work_products;
use rbml::opaque::Encoder;
use rustc::dep_graph::DepNode;
use rustc::middle::cstore::LOCAL_CRATE;
+use rustc::session::Session;
use rustc::ty::TyCtxt;
use rustc_serialize::{Encodable as RustcEncodable};
use std::hash::{Hasher, SipHasher};
use super::util::*;
pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
+ debug!("save_dep_graph()");
let _ignore = tcx.dep_graph.in_ignore();
+ let sess = tcx.sess;
let mut hcx = HashContext::new(tcx);
- save_in(&mut hcx, dep_graph_path(tcx), encode_dep_graph);
- save_in(&mut hcx, metadata_hash_path(tcx, LOCAL_CRATE), encode_metadata_hashes);
+ save_in(sess, dep_graph_path(tcx), |e| encode_dep_graph(&mut hcx, e));
+ save_in(sess, metadata_hash_path(tcx, LOCAL_CRATE), |e| encode_metadata_hashes(&mut hcx, e));
}
-fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
- opt_path_buf: Option<PathBuf>,
- encode: F)
- where F: FnOnce(&mut HashContext<'a, 'tcx>, &mut Encoder) -> io::Result<()>
-{
- let tcx = hcx.tcx;
+pub fn save_work_products(sess: &Session, local_crate_name: &str) {
+ debug!("save_work_products()");
+ let _ignore = sess.dep_graph.in_ignore();
+ let path = sess_work_products_path(sess, local_crate_name);
+ save_in(sess, path, |e| encode_work_products(sess, e));
+}
+fn save_in<F>(sess: &Session,
+ opt_path_buf: Option<PathBuf>,
+ encode: F)
+ where F: FnOnce(&mut Encoder) -> io::Result<()>
+{
let path_buf = match opt_path_buf {
Some(p) => p,
None => return
match fs::remove_file(&path_buf) {
Ok(()) => { }
Err(err) => {
- tcx.sess.err(
+ sess.err(
&format!("unable to delete old dep-graph at `{}`: {}",
path_buf.display(), err));
return;
// generate the data in a memory buffer
let mut wr = Cursor::new(Vec::new());
- match encode(hcx, &mut Encoder::new(&mut wr)) {
+ match encode(&mut Encoder::new(&mut wr)) {
Ok(()) => { }
Err(err) => {
- tcx.sess.err(
+ sess.err(
&format!("could not encode dep-graph to `{}`: {}",
path_buf.display(), err));
return;
{
Ok(_) => { }
Err(err) => {
- tcx.sess.err(
+ sess.err(
&format!("failed to write dep-graph to `{}`: {}",
path_buf.display(), err));
return;
Ok(())
}
+
+pub fn encode_work_products(sess: &Session,
+ encoder: &mut Encoder)
+ -> io::Result<()>
+{
+ let work_products: Vec<_> =
+ sess.dep_graph.work_products()
+ .iter()
+ .map(|(id, work_product)| {
+ SerializedWorkProduct {
+ id: id.clone(),
+ work_product: work_product.clone(),
+ }
+ })
+ .collect();
+
+ work_products.encode(encoder)
+}
+
// except according to those terms.
use rustc::middle::cstore::LOCAL_CRATE;
+use rustc::session::Session;
use rustc::ty::TyCtxt;
use std::fs;
use syntax::ast;
pub fn dep_graph_path(tcx: TyCtxt) -> Option<PathBuf> {
- path(tcx, LOCAL_CRATE, "local")
+ tcx_path(tcx, LOCAL_CRATE, "local")
}
pub fn metadata_hash_path(tcx: TyCtxt, cnum: ast::CrateNum) -> Option<PathBuf> {
- path(tcx, cnum, "metadata")
+ tcx_path(tcx, cnum, "metadata")
}
-fn path(tcx: TyCtxt, cnum: ast::CrateNum, suffix: &str) -> Option<PathBuf> {
+pub fn tcx_work_products_path(tcx: TyCtxt) -> Option<PathBuf> {
+ let crate_name = tcx.crate_name(LOCAL_CRATE);
+ sess_work_products_path(tcx.sess, &crate_name)
+}
+
+pub fn sess_work_products_path(sess: &Session,
+ local_crate_name: &str)
+ -> Option<PathBuf> {
+ let crate_disambiguator = sess.local_crate_disambiguator();
+ path(sess, local_crate_name, &crate_disambiguator, "work-products")
+}
+
+pub fn in_incr_comp_dir(sess: &Session, file_name: &str) -> Option<PathBuf> {
+ sess.opts.incremental.as_ref().map(|incr_dir| incr_dir.join(file_name))
+}
+
+fn tcx_path(tcx: TyCtxt,
+ cnum: ast::CrateNum,
+ middle: &str)
+ -> Option<PathBuf> {
+ path(tcx.sess, &tcx.crate_name(cnum), &tcx.crate_disambiguator(cnum), middle)
+}
+
+fn path(sess: &Session,
+ crate_name: &str,
+ crate_disambiguator: &str,
+ middle: &str)
+ -> Option<PathBuf> {
// For now, just save/load dep-graph from
// directory/dep_graph.rbml
- tcx.sess.opts.incremental.as_ref().and_then(|incr_dir| {
+ sess.opts.incremental.as_ref().and_then(|incr_dir| {
match create_dir_racy(&incr_dir) {
Ok(()) => {}
Err(err) => {
- tcx.sess.err(
+ sess.err(
&format!("could not create the directory `{}`: {}",
incr_dir.display(), err));
return None;
}
}
- let crate_name = tcx.crate_name(cnum);
- let crate_disambiguator = tcx.crate_disambiguator(cnum);
- let file_name = format!("{}-{}.{}.bin",
- crate_name,
- crate_disambiguator,
- suffix);
+ let file_name = format!("{}-{}.{}.bin", crate_name, crate_disambiguator, middle);
+
Some(incr_dir.join(file_name))
})
}
projects / rust.git / commitdiff