clippy_dev/src/crater.rs

   1 use crate::clippy_project_root;
   2 use std::path::PathBuf;
   3 use std::process::Command;
   4 // represents an archive we download from crates.io
   5 #[derive(Debug)]
   6 struct KrateSource {
   7     version: String,
   8     name: String,
   9 }
  10
  11 // represents the extracted sourcecode of a crate
  12 #[derive(Debug)]
  13 struct Krate {
  14     version: String,
  15     name: String,
  16     // path to the extracted sources that clippy can check
  17     path: PathBuf,
  18 }
  19
  20 impl KrateSource {
  21     fn new(name: &str, version: &str) -> Self {
  22         KrateSource {
  23             version: version.into(),
  24             name: name.into(),
  25         }
  26     }
  27
  28     fn download_and_extract(&self) -> Krate {
  29         let extract_dir = PathBuf::from("target/crater/crates");
  30         let krate_download_dir = PathBuf::from("target/crater/downloads");
  31
  32         // url to download the crate from crates.io
  33         let url = format!(
  34             "https://crates.io/api/v1/crates/{}/{}/download",
  35             self.name, self.version
  36         );
  37         println!("Downloading and extracting {} {} from {}", self.name, self.version, url);
  38         let _ = std::fs::create_dir("target/crater/");
  39         let _ = std::fs::create_dir(&krate_download_dir);
  40         let _ = std::fs::create_dir(&extract_dir);
  41
  42         let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", &self.name, &self.version));
  43         // don't download/extract if we already have done so
  44         if !krate_file_path.is_file() {
  45             // create a file path to download and write the crate data into
  46             let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
  47             let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
  48             // copy the crate into the file
  49             std::io::copy(&mut krate_req, &mut krate_dest).unwrap();
  50
  51             // unzip the tarball
  52             let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
  53             // extract the tar archive
  54             let mut archiv = tar::Archive::new(ungz_tar);
  55             archiv.unpack(&extract_dir).expect("Failed to extract!");
  56
  57             // unzip the tarball
  58             let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
  59             // extract the tar archive
  60             let mut archiv = tar::Archive::new(ungz_tar);
  61             archiv.unpack(&extract_dir).expect("Failed to extract!");
  62         }
  63         // crate is extracted, return a new Krate object which contains the path to the extracted
  64         // sources that clippy can check
  65         Krate {
  66             version: self.version.clone(),
  67             name: self.name.clone(),
  68             path: extract_dir.join(format!("{}-{}/", self.name, self.version)),
  69         }
  70     }
  71 }
  72
  73 impl Krate {
  74     fn run_clippy_lints(&self, cargo_clippy_path: &PathBuf) -> Vec<String> {
  75         println!("Linting {} {}...", &self.name, &self.version);
  76         let cargo_clippy_path = std::fs::canonicalize(cargo_clippy_path).unwrap();
  77
  78         let shared_target_dir = clippy_project_root().join("target/crater/shared_target_dir/");
  79
  80         let all_output = std::process::Command::new(cargo_clippy_path)
  81             .env("CARGO_TARGET_DIR", shared_target_dir)
  82             // lint warnings will look like this:
  83             // src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
  84             .args(&[
  85                 "--",
  86                 "--message-format=short",
  87                 "--",
  88                 "--cap-lints=warn",
  89                 "-Wclippy::pedantic",
  90                 "-Wclippy::cargo",
  91             ])
  92             .current_dir(&self.path)
  93             .output()
  94             .unwrap();
  95         let stderr = String::from_utf8_lossy(&all_output.stderr);
  96         let output_lines = stderr.lines();
  97         let mut output: Vec<String> = output_lines
  98             .into_iter()
  99             .filter(|line| line.contains(": warning: "))
 100             // prefix with the crate name and version
 101             // cargo-0.49.0/src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
 102             .map(|line| format!("{}-{}/{}", self.name, self.version, line))
 103             // remove the "warning: "
 104             .map(|line| {
 105                 let remove_pat = "warning: ";
 106                 let pos = line
 107                     .find(&remove_pat)
 108                     .expect("clippy output did not contain \"warning: \"");
 109                 let mut new = line[0..pos].to_string();
 110                 new.push_str(&line[pos + remove_pat.len()..]);
 111                 new.push('\n');
 112                 new
 113             })
 114             .collect();
 115
 116         // sort messages alphabtically to avoid noise in the logs
 117         output.sort();
 118         output
 119     }
 120 }
 121
 122 fn build_clippy() {
 123     Command::new("cargo")
 124         .arg("build")
 125         .output()
 126         .expect("Failed to build clippy!");
 127 }
 128
 129 // the main fn
 130 pub fn run() {
 131     let cargo_clippy_path: PathBuf = PathBuf::from("target/debug/cargo-clippy");
 132
 133     // crates we want to check:
 134     let krates: Vec<KrateSource> = vec![
 135         // some of these are form cargotest
 136         KrateSource::new("cargo", "0.49.0"),
 137         KrateSource::new("iron", "0.6.1"),
 138         KrateSource::new("ripgrep", "12.1.1"),
 139         //KrateSource::new("tokei", "12.0.4"),
 140         KrateSource::new("xsv", "0.13.0"),
 141         KrateSource::new("serde", "1.0.118"),
 142         KrateSource::new("rayon", "1.5.0"),
 143         // top 10 crates.io dls
 144         KrateSource::new("rand", "0.7.3"),
 145         KrateSource::new("syn", "1.0.54"),
 146         KrateSource::new("libc", "0.2.81"),
 147         KrateSource::new("quote", "1.0.7"),
 148         KrateSource::new("rand_core", "0.6.0"),
 149         KrateSource::new("unicode-xid", "0.2.1"),
 150         KrateSource::new("proc-macro2", "1.0.24"),
 151         KrateSource::new("bitflags", "1.2.1"),
 152         KrateSource::new("log", "0.4.11"),
 153         KrateSource::new("regex", "1.4.2"),
 154     ];
 155
 156     println!("Compiling clippy...");
 157     build_clippy();
 158     println!("Done compiling");
 159
 160     // assert that clippy is found
 161     assert!(
 162         cargo_clippy_path.is_file(),
 163         "target/debug/cargo-clippy binary not found! {}",
 164         cargo_clippy_path.display()
 165     );
 166
 167     // download and extract the crates, then run clippy on them and collect clippys warnings
 168     let clippy_lint_results: Vec<Vec<String>> = krates
 169         .into_iter()
 170         .map(|krate| krate.download_and_extract())
 171         .map(|krate| krate.run_clippy_lints(&cargo_clippy_path))
 172         .collect();
 173
 174     let all_warnings: Vec<String> = clippy_lint_results.into_iter().flatten().collect();
 175
 176     // save the text into mini-crater/logs.txt
 177     let text = all_warnings.join("");
 178     std::fs::write("mini-crater/logs.txt", text).unwrap();
 179 }