src/tools/tidy/src/features.rs

   1 //! Tidy check to ensure that unstable features are all in order.
   2 //!
   3 //! This check will ensure properties like:
   4 //!
   5 //! * All stability attributes look reasonably well formed.
   6 //! * The set of library features is disjoint from the set of language features.
   7 //! * Library features have at most one stability level.
   8 //! * Library features have at most one `since` value.
   9 //! * All unstable lang features have tests to ensure they are actually unstable.
  10 //! * Language features in a group are sorted by `since` value.
  11
  12 use std::collections::HashMap;
  13 use std::fmt;
  14 use std::fs;
  15 use std::path::Path;
  16
  17 use regex::Regex;
  18
  19 #[cfg(test)]
  20 mod tests;
  21
  22 mod version;
  23 use version::Version;
  24
  25 const FEATURE_GROUP_START_PREFIX: &str = "// feature-group-start";
  26 const FEATURE_GROUP_END_PREFIX: &str = "// feature-group-end";
  27
  28 #[derive(Debug, PartialEq, Clone)]
  29 pub enum Status {
  30     Stable,
  31     Removed,
  32     Unstable,
  33 }
  34
  35 impl fmt::Display for Status {
  36     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  37         let as_str = match *self {
  38             Status::Stable => "stable",
  39             Status::Unstable => "unstable",
  40             Status::Removed => "removed",
  41         };
  42         fmt::Display::fmt(as_str, f)
  43     }
  44 }
  45
  46 #[derive(Debug, Clone)]
  47 pub struct Feature {
  48     pub level: Status,
  49     pub since: Option<Version>,
  50     pub has_gate_test: bool,
  51     pub tracking_issue: Option<u32>,
  52 }
  53
  54 pub type Features = HashMap<String, Feature>;
  55
  56 pub struct CollectedFeatures {
  57     pub lib: Features,
  58     pub lang: Features,
  59 }
  60
  61 // Currently only used for unstable book generation
  62 pub fn collect_lib_features(base_src_path: &Path) -> Features {
  63     let mut lib_features = Features::new();
  64
  65     // This library feature is defined in the `compiler_builtins` crate, which
  66     // has been moved out-of-tree. Now it can no longer be auto-discovered by
  67     // `tidy`, because we need to filter out its (submodule) directory. Manually
  68     // add it to the set of known library features so we can still generate docs.
  69     lib_features.insert("compiler_builtins_lib".to_owned(), Feature {
  70         level: Status::Unstable,
  71         since: None,
  72         has_gate_test: false,
  73         tracking_issue: None,
  74     });
  75
  76     map_lib_features(base_src_path,
  77                      &mut |res, _, _| {
  78         if let Ok((name, feature)) = res {
  79             lib_features.insert(name.to_owned(), feature);
  80         }
  81     });
  82    lib_features
  83 }
  84
  85 pub fn check(path: &Path, bad: &mut bool, verbose: bool) -> CollectedFeatures {
  86     let mut features = collect_lang_features(path, bad);
  87     assert!(!features.is_empty());
  88
  89     let lib_features = get_and_check_lib_features(path, bad, &features);
  90     assert!(!lib_features.is_empty());
  91
  92     super::walk_many(&[&path.join("test/ui"),
  93                        &path.join("test/ui-fulldeps"),
  94                        &path.join("test/compile-fail")],
  95                      &mut |path| super::filter_dirs(path),
  96                      &mut |entry, contents| {
  97         let file = entry.path();
  98         let filename = file.file_name().unwrap().to_string_lossy();
  99         if !filename.ends_with(".rs") || filename == "features.rs" ||
 100            filename == "diagnostic_list.rs" {
 101             return;
 102         }
 103
 104         let filen_underscore = filename.replace('-',"_").replace(".rs","");
 105         let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);
 106
 107         for (i, line) in contents.lines().enumerate() {
 108             let mut err = |msg: &str| {
 109                 tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
 110             };
 111
 112             let gate_test_str = "gate-test-";
 113
 114             let feature_name = match line.find(gate_test_str) {
 115                 Some(i) => {
 116                     line[i+gate_test_str.len()..].splitn(2, ' ').next().unwrap()
 117                 },
 118                 None => continue,
 119             };
 120             match features.get_mut(feature_name) {
 121                 Some(f) => {
 122                     if filename_is_gate_test {
 123                         err(&format!("The file is already marked as gate test \
 124                                       through its name, no need for a \
 125                                       'gate-test-{}' comment",
 126                                      feature_name));
 127                     }
 128                     f.has_gate_test = true;
 129                 }
 130                 None => {
 131                     err(&format!("gate-test test found referencing a nonexistent feature '{}'",
 132                                  feature_name));
 133                 }
 134             }
 135         }
 136     });
 137
 138     // Only check the number of lang features.
 139     // Obligatory testing for library features is dumb.
 140     let gate_untested = features.iter()
 141                                 .filter(|&(_, f)| f.level == Status::Unstable)
 142                                 .filter(|&(_, f)| !f.has_gate_test)
 143                                 .collect::<Vec<_>>();
 144
 145     for &(name, _) in gate_untested.iter() {
 146         println!("Expected a gate test for the feature '{}'.", name);
 147         println!("Hint: create a failing test file named 'feature-gate-{}.rs'\
 148                 \n      in the 'ui' test suite, with its failures due to\
 149                 \n      missing usage of `#![feature({})]`.", name, name);
 150         println!("Hint: If you already have such a test and don't want to rename it,\
 151                 \n      you can also add a // gate-test-{} line to the test file.",
 152                  name);
 153     }
 154
 155     if !gate_untested.is_empty() {
 156         tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
 157     }
 158
 159     if *bad {
 160         return CollectedFeatures { lib: lib_features, lang: features };
 161     }
 162
 163     if verbose {
 164         let mut lines = Vec::new();
 165         lines.extend(format_features(&features, "lang"));
 166         lines.extend(format_features(&lib_features, "lib"));
 167
 168         lines.sort();
 169         for line in lines {
 170             println!("* {}", line);
 171         }
 172     } else {
 173         println!("* {} features", features.len());
 174     }
 175
 176     CollectedFeatures { lib: lib_features, lang: features }
 177 }
 178
 179 fn format_features<'a>(
 180     features: &'a Features,
 181     family: &'a str,
 182 ) -> impl Iterator<Item = String> + 'a {
 183     features.iter().map(move |(name, feature)| {
 184         format!("{:<32} {:<8} {:<12} {:<8}",
 185                 name,
 186                 family,
 187                 feature.level,
 188                 feature.since.map_or("None".to_owned(),
 189                                      |since| since.to_string()))
 190     })
 191 }
 192
 193 fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
 194     lazy_static::lazy_static! {
 195         static ref ISSUE: Regex = Regex::new(r#"issue\s*=\s*"([^"]*)""#).unwrap();
 196         static ref FEATURE: Regex = Regex::new(r#"feature\s*=\s*"([^"]*)""#).unwrap();
 197         static ref SINCE: Regex = Regex::new(r#"since\s*=\s*"([^"]*)""#).unwrap();
 198     }
 199
 200     let r = match attr {
 201         "issue" => &*ISSUE,
 202         "feature" => &*FEATURE,
 203         "since" => &*SINCE,
 204         _ => unimplemented!("{} not handled", attr),
 205     };
 206
 207     r.captures(line)
 208         .and_then(|c| c.get(1))
 209         .map(|m| m.as_str())
 210 }
 211
 212 fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
 213     let prefix = "feature_gate_";
 214     if filen_underscore.starts_with(prefix) {
 215         for (n, f) in features.iter_mut() {
 216             // Equivalent to filen_underscore == format!("feature_gate_{}", n)
 217             if &filen_underscore[prefix.len()..] == n {
 218                 f.has_gate_test = true;
 219                 return true;
 220             }
 221         }
 222     }
 223     return false;
 224 }
 225
 226 pub fn collect_lang_features(base_src_path: &Path, bad: &mut bool) -> Features {
 227     let mut all = collect_lang_features_in(base_src_path, "active.rs", bad);
 228     all.extend(collect_lang_features_in(base_src_path, "accepted.rs", bad));
 229     all.extend(collect_lang_features_in(base_src_path, "removed.rs", bad));
 230     all
 231 }
 232
 233 fn collect_lang_features_in(base: &Path, file: &str, bad: &mut bool) -> Features {
 234     let path = base.join("libsyntax/feature_gate").join(file);
 235     let contents = t!(fs::read_to_string(&path));
 236
 237     // We allow rustc-internal features to omit a tracking issue.
 238     // To make tidy accept omitting a tracking issue, group the list of features
 239     // without one inside `// no-tracking-issue` and `// no-tracking-issue-end`.
 240     let mut next_feature_omits_tracking_issue = false;
 241
 242     let mut in_feature_group = false;
 243     let mut prev_since = None;
 244
 245     contents.lines().zip(1..)
 246         .filter_map(|(line, line_number)| {
 247             let line = line.trim();
 248
 249             // Within -start and -end, the tracking issue can be omitted.
 250             match line {
 251                 "// no-tracking-issue-start" => {
 252                     next_feature_omits_tracking_issue = true;
 253                     return None;
 254                 }
 255                 "// no-tracking-issue-end" => {
 256                     next_feature_omits_tracking_issue = false;
 257                     return None;
 258                 }
 259                 _ => {}
 260             }
 261
 262             if line.starts_with(FEATURE_GROUP_START_PREFIX) {
 263                 if in_feature_group {
 264                     tidy_error!(
 265                         bad,
 266                         "{}:{}: \
 267                         new feature group is started without ending the previous one",
 268                         path.display(),
 269                         line_number,
 270                     );
 271                 }
 272
 273                 in_feature_group = true;
 274                 prev_since = None;
 275                 return None;
 276             } else if line.starts_with(FEATURE_GROUP_END_PREFIX) {
 277                 in_feature_group = false;
 278                 prev_since = None;
 279                 return None;
 280             }
 281
 282             let mut parts = line.split(',');
 283             let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {
 284                 Some("active") => Status::Unstable,
 285                 Some("removed") => Status::Removed,
 286                 Some("accepted") => Status::Stable,
 287                 _ => return None,
 288             };
 289             let name = parts.next().unwrap().trim();
 290
 291             let since_str = parts.next().unwrap().trim().trim_matches('"');
 292             let since = match since_str.parse() {
 293                 Ok(since) => Some(since),
 294                 Err(err) => {
 295                     tidy_error!(
 296                         bad,
 297                         "{}:{}: failed to parse since: {} ({:?})",
 298                         path.display(),
 299                         line_number,
 300                         since_str,
 301                         err,
 302                     );
 303                     None
 304                 }
 305             };
 306             if in_feature_group {
 307                 if prev_since > since {
 308                     tidy_error!(
 309                         bad,
 310                         "{}:{}: feature {} is not sorted by since",
 311                         path.display(),
 312                         line_number,
 313                         name,
 314                     );
 315                 }
 316                 prev_since = since;
 317             }
 318
 319             let issue_str = parts.next().unwrap().trim();
 320             let tracking_issue = if issue_str.starts_with("None") {
 321                 if level == Status::Unstable && !next_feature_omits_tracking_issue {
 322                     *bad = true;
 323                     tidy_error!(
 324                         bad,
 325                         "{}:{}: no tracking issue for feature {}",
 326                         path.display(),
 327                         line_number,
 328                         name,
 329                     );
 330                 }
 331                 None
 332             } else {
 333                 let s = issue_str.split('(').nth(1).unwrap().split(')').nth(0).unwrap();
 334                 Some(s.parse().unwrap())
 335             };
 336             Some((name.to_owned(),
 337                 Feature {
 338                     level,
 339                     since,
 340                     has_gate_test: false,
 341                     tracking_issue,
 342                 }))
 343         })
 344         .collect()
 345 }
 346
 347 fn get_and_check_lib_features(base_src_path: &Path,
 348                               bad: &mut bool,
 349                               lang_features: &Features) -> Features {
 350     let mut lib_features = Features::new();
 351     map_lib_features(base_src_path,
 352                      &mut |res, file, line| {
 353             match res {
 354                 Ok((name, f)) => {
 355                     let mut check_features = |f: &Feature, list: &Features, display: &str| {
 356                         if let Some(ref s) = list.get(name) {
 357                             if f.tracking_issue != s.tracking_issue && f.level != Status::Stable {
 358                                 tidy_error!(bad,
 359                                             "{}:{}: mismatches the `issue` in {}",
 360                                             file.display(),
 361                                             line,
 362                                             display);
 363                             }
 364                         }
 365                     };
 366                     check_features(&f, &lang_features, "corresponding lang feature");
 367                     check_features(&f, &lib_features, "previous");
 368                     lib_features.insert(name.to_owned(), f);
 369                 },
 370                 Err(msg) => {
 371                     tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
 372                 },
 373             }
 374
 375     });
 376     lib_features
 377 }
 378
 379 fn map_lib_features(base_src_path: &Path,
 380                     mf: &mut dyn FnMut(Result<(&str, Feature), &str>, &Path, usize)) {
 381     super::walk(base_src_path,
 382                 &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
 383                 &mut |entry, contents| {
 384         let file = entry.path();
 385         let filename = file.file_name().unwrap().to_string_lossy();
 386         if !filename.ends_with(".rs") || filename == "features.rs" ||
 387            filename == "diagnostic_list.rs" {
 388             return;
 389         }
 390
 391         // This is an early exit -- all the attributes we're concerned with must contain this:
 392         // * rustc_const_unstable(
 393         // * unstable(
 394         // * stable(
 395         if !contents.contains("stable(") {
 396             return;
 397         }
 398
 399         let mut becoming_feature: Option<(&str, Feature)> = None;
 400         for (i, line) in contents.lines().enumerate() {
 401             macro_rules! err {
 402                 ($msg:expr) => {{
 403                     mf(Err($msg), file, i + 1);
 404                     continue;
 405                 }};
 406             };
 407             if let Some((ref name, ref mut f)) = becoming_feature {
 408                 if f.tracking_issue.is_none() {
 409                     f.tracking_issue = find_attr_val(line, "issue")
 410                     .map(|s| s.parse().unwrap());
 411                 }
 412                 if line.ends_with(']') {
 413                     mf(Ok((name, f.clone())), file, i + 1);
 414                 } else if !line.ends_with(',') && !line.ends_with('\\') {
 415                     // We need to bail here because we might have missed the
 416                     // end of a stability attribute above because the ']'
 417                     // might not have been at the end of the line.
 418                     // We could then get into the very unfortunate situation that
 419                     // we continue parsing the file assuming the current stability
 420                     // attribute has not ended, and ignoring possible feature
 421                     // attributes in the process.
 422                     err!("malformed stability attribute");
 423                 } else {
 424                     continue;
 425                 }
 426             }
 427             becoming_feature = None;
 428             if line.contains("rustc_const_unstable(") {
 429                 // `const fn` features are handled specially.
 430                 let feature_name = match find_attr_val(line, "feature") {
 431                     Some(name) => name,
 432                     None => err!("malformed stability attribute: missing `feature` key"),
 433                 };
 434                 let feature = Feature {
 435                     level: Status::Unstable,
 436                     since: None,
 437                     has_gate_test: false,
 438                     // FIXME(#57563): #57563 is now used as a common tracking issue,
 439                     // although we would like to have specific tracking issues for each
 440                     // `rustc_const_unstable` in the future.
 441                     tracking_issue: Some(57563),
 442                 };
 443                 mf(Ok((feature_name, feature)), file, i + 1);
 444                 continue;
 445             }
 446             let level = if line.contains("[unstable(") {
 447                 Status::Unstable
 448             } else if line.contains("[stable(") {
 449                 Status::Stable
 450             } else {
 451                 continue;
 452             };
 453             let feature_name = match find_attr_val(line, "feature") {
 454                 Some(name) => name,
 455                 None => err!("malformed stability attribute: missing `feature` key"),
 456             };
 457             let since = match find_attr_val(line, "since").map(|x| x.parse()) {
 458                 Some(Ok(since)) => Some(since),
 459                 Some(Err(_err)) => {
 460                     err!("malformed stability attribute: can't parse `since` key");
 461                 },
 462                 None if level == Status::Stable => {
 463                     err!("malformed stability attribute: missing the `since` key");
 464                 }
 465                 None => None,
 466             };
 467             let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());
 468
 469             let feature = Feature {
 470                 level,
 471                 since,
 472                 has_gate_test: false,
 473                 tracking_issue,
 474             };
 475             if line.contains(']') {
 476                 mf(Ok((feature_name, feature)), file, i + 1);
 477             } else {
 478                 becoming_feature = Some((feature_name, feature));
 479             }
 480         }
 481     });
 482 }