From 73141337223bfa55d03f2d1097af6a036a6cbbc5 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Matthias=20Kr=C3=BCger?= Date: Fri, 18 Dec 2020 20:58:46 +0100 Subject: [PATCH] cargo dev crater: cleanup, don't re-download and reextract crates on every run --- clippy_dev/src/crater.rs | 88 ++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/clippy_dev/src/crater.rs b/clippy_dev/src/crater.rs index 6b121a79e37..63c04f9a1f1 100644 --- a/clippy_dev/src/crater.rs +++ b/clippy_dev/src/crater.rs @@ -13,6 +13,7 @@ struct KrateSource { struct Krate { version: String, name: String, + // path to the extracted sources that clippy can check path: PathBuf, } @@ -23,37 +24,44 @@ fn new(name: &str, version: &str) -> Self { name: name.into(), } } + fn download_and_extract(&self) -> Krate { let extract_dir = PathBuf::from("target/crater/crates"); - - // download let krate_download_dir = PathBuf::from("target/crater/downloads"); + // url to download the crate from crates.io let url = format!( "https://crates.io/api/v1/crates/{}/{}/download", self.name, self.version ); - println!("Downloading {}, {} / {}", self.name, self.version, url); + println!("Downloading and extracting {} {} from {}", self.name, self.version, url); let _ = std::fs::create_dir("target/crater/"); - let _ = std::fs::create_dir(&krate_download_dir); let _ = std::fs::create_dir(&extract_dir); - let krate_name = format!("{}-{}.crate.tar.gz", &self.name, &self.version); - let krate_file_path = krate_download_dir.join(krate_name); - let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap(); - let mut krate_req = ureq::get(&url).call().unwrap().into_reader(); - std::io::copy(&mut krate_req, &mut krate_dest).unwrap(); - // unzip the tarball - let dl = std::fs::File::open(krate_file_path).unwrap(); - - let ungz_tar = flate2::read::GzDecoder::new(dl); - // extract the tar archive - let mut archiv = tar::Archive::new(ungz_tar); - let extract_path = extract_dir.clone(); - archiv.unpack(&extract_path).expect("Failed to extract!"); - // extracted - dbg!(&extract_path); + let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", &self.name, &self.version)); + // don't download/extract if we already have done so + if !krate_file_path.is_file() { + // create a file path to download and write the crate data into + let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap(); + let mut krate_req = ureq::get(&url).call().unwrap().into_reader(); + // copy the crate into the file + std::io::copy(&mut krate_req, &mut krate_dest).unwrap(); + + // unzip the tarball + let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap()); + // extract the tar archive + let mut archiv = tar::Archive::new(ungz_tar); + archiv.unpack(&extract_dir).expect("Failed to extract!"); + + // unzip the tarball + let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap()); + // extract the tar archive + let mut archiv = tar::Archive::new(ungz_tar); + archiv.unpack(&extract_dir).expect("Failed to extract!"); + } + // crate is extracted, return a new Krate object which contains the path to the extracted + // sources that clippy can check Krate { version: self.version.clone(), name: self.name.clone(), @@ -64,24 +72,37 @@ fn download_and_extract(&self) -> Krate { impl Krate { fn run_clippy_lints(&self, cargo_clippy_path: &PathBuf) -> Vec { + println!("Linting {} {}...", &self.name, &self.version); let cargo_clippy_path = std::fs::canonicalize(cargo_clippy_path).unwrap(); - let project_root = &self.path; - dbg!(&cargo_clippy_path); - dbg!(&project_root); - let output = std::process::Command::new(cargo_clippy_path) + let all_output = std::process::Command::new(cargo_clippy_path) + // lint warnings will look like this: + // src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter` .args(&["--", "--message-format=short", "--", "--cap-lints=warn"]) - .current_dir(project_root) + .current_dir(&self.path) .output() .unwrap(); - let mut output = String::from_utf8_lossy(&output.stderr); - let output: Vec<&str> = output.lines().collect(); - let mut output: Vec = output + let stderr = String::from_utf8_lossy(&all_output.stderr); + let output_lines = stderr.lines(); + let mut output: Vec = output_lines .into_iter() .filter(|line| line.contains(": warning: ")) - .map(|l| l.to_string()) + // prefix with the crate name and version + // cargo-0.49.0/src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter` + .map(|line| format!("{}-{}/{}", self.name, self.version, line)) + // remove the "warning: " + .map(|line| { + let remove_pat = "warning: "; + let pos = line + .find(&remove_pat) + .expect("clippy output did not contain \"warning: \""); + let mut new = line[0..pos].to_string(); + new.push_str(&line[pos + remove_pat.len()..]); + new + }) .collect(); + // sort messages alphabtically to avoid noise in the logs output.sort(); output } @@ -97,7 +118,6 @@ fn build_clippy() { // the main fn pub fn run() { let cargo_clippy_path: PathBuf = PathBuf::from("target/debug/cargo-clippy"); - let clippy_driver_path: PathBuf = PathBuf::from("target/debug/clippy-driver"); // crates we want to check: let krates: Vec = vec![KrateSource::new("regex", "1.4.2"), KrateSource::new("cargo", "0.49.0")]; @@ -112,11 +132,6 @@ pub fn run() { "target/debug/cargo-clippy binary not found! {}", cargo_clippy_path.display() ); - assert!( - clippy_driver_path.is_file(), - "target/debug/clippy-driver binary not found! {}", - clippy_driver_path.display() - ); // download and extract the crates, then run clippy on them and collect clippys warnings let clippy_lint_results: Vec> = krates @@ -125,7 +140,8 @@ pub fn run() { .map(|krate| krate.run_clippy_lints(&cargo_clippy_path)) .collect(); - let results: Vec = clippy_lint_results.into_iter().flatten().collect(); + let all_warnings: Vec = clippy_lint_results.into_iter().flatten().collect(); - results.iter().for_each(|l| println!("{}", l)); + // TODO: save these into a file + all_warnings.iter().for_each(|l| println!("{}", l)); } -- 2.44.0