src/tools/tidy/src/features.rs

   1 //! Tidy check to ensure that unstable features are all in order.
   2 //!
   3 //! This check will ensure properties like:
   4 //!
   5 //! * All stability attributes look reasonably well formed.
   6 //! * The set of library features is disjoint from the set of language features.
   7 //! * Library features have at most one stability level.
   8 //! * Library features have at most one `since` value.
   9 //! * All unstable lang features have tests to ensure they are actually unstable.
  10 //! * Language features in a group are sorted by feature name.
  11
  12 use crate::walk::{filter_dirs, walk, walk_many};
  13 use std::collections::hash_map::{Entry, HashMap};
  14 use std::fmt;
  15 use std::fs;
  16 use std::num::NonZeroU32;
  17 use std::path::Path;
  18
  19 use regex::Regex;
  20
  21 #[cfg(test)]
  22 mod tests;
  23
  24 mod version;
  25 use version::Version;
  26
  27 const FEATURE_GROUP_START_PREFIX: &str = "// feature-group-start";
  28 const FEATURE_GROUP_END_PREFIX: &str = "// feature-group-end";
  29
  30 #[derive(Debug, PartialEq, Clone)]
  31 pub enum Status {
  32     Stable,
  33     Removed,
  34     Unstable,
  35 }
  36
  37 impl fmt::Display for Status {
  38     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  39         let as_str = match *self {
  40             Status::Stable => "stable",
  41             Status::Unstable => "unstable",
  42             Status::Removed => "removed",
  43         };
  44         fmt::Display::fmt(as_str, f)
  45     }
  46 }
  47
  48 #[derive(Debug, Clone)]
  49 pub struct Feature {
  50     pub level: Status,
  51     pub since: Option<Version>,
  52     pub has_gate_test: bool,
  53     pub tracking_issue: Option<NonZeroU32>,
  54 }
  55 impl Feature {
  56     fn tracking_issue_display(&self) -> impl fmt::Display {
  57         match self.tracking_issue {
  58             None => "none".to_string(),
  59             Some(x) => x.to_string(),
  60         }
  61     }
  62 }
  63
  64 pub type Features = HashMap<String, Feature>;
  65
  66 pub struct CollectedFeatures {
  67     pub lib: Features,
  68     pub lang: Features,
  69 }
  70
  71 // Currently only used for unstable book generation
  72 pub fn collect_lib_features(base_src_path: &Path) -> Features {
  73     let mut lib_features = Features::new();
  74
  75     map_lib_features(base_src_path, &mut |res, _, _| {
  76         if let Ok((name, feature)) = res {
  77             lib_features.insert(name.to_owned(), feature);
  78         }
  79     });
  80     lib_features
  81 }
  82
  83 pub fn check(
  84     src_path: &Path,
  85     compiler_path: &Path,
  86     lib_path: &Path,
  87     bad: &mut bool,
  88     verbose: bool,
  89 ) -> CollectedFeatures {
  90     let mut features = collect_lang_features(compiler_path, bad);
  91     assert!(!features.is_empty());
  92
  93     let lib_features = get_and_check_lib_features(lib_path, bad, &features);
  94     assert!(!lib_features.is_empty());
  95
  96     walk_many(
  97         &[
  98             &src_path.join("test/ui"),
  99             &src_path.join("test/ui-fulldeps"),
 100             &src_path.join("test/rustdoc-ui"),
 101             &src_path.join("test/rustdoc"),
 102         ],
 103         &mut filter_dirs,
 104         &mut |entry, contents| {
 105             let file = entry.path();
 106             let filename = file.file_name().unwrap().to_string_lossy();
 107             if !filename.ends_with(".rs")
 108                 || filename == "features.rs"
 109                 || filename == "diagnostic_list.rs"
 110             {
 111                 return;
 112             }
 113
 114             let filen_underscore = filename.replace('-', "_").replace(".rs", "");
 115             let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);
 116
 117             for (i, line) in contents.lines().enumerate() {
 118                 let mut err = |msg: &str| {
 119                     tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
 120                 };
 121
 122                 let gate_test_str = "gate-test-";
 123
 124                 let feature_name = match line.find(gate_test_str) {
 125                     // NB: the `splitn` always succeeds, even if the delimiter is not present.
 126                     Some(i) => line[i + gate_test_str.len()..].splitn(2, ' ').next().unwrap(),
 127                     None => continue,
 128                 };
 129                 match features.get_mut(feature_name) {
 130                     Some(f) => {
 131                         if filename_is_gate_test {
 132                             err(&format!(
 133                                 "The file is already marked as gate test \
 134                                       through its name, no need for a \
 135                                       'gate-test-{}' comment",
 136                                 feature_name
 137                             ));
 138                         }
 139                         f.has_gate_test = true;
 140                     }
 141                     None => {
 142                         err(&format!(
 143                             "gate-test test found referencing a nonexistent feature '{}'",
 144                             feature_name
 145                         ));
 146                     }
 147                 }
 148             }
 149         },
 150     );
 151
 152     // Only check the number of lang features.
 153     // Obligatory testing for library features is dumb.
 154     let gate_untested = features
 155         .iter()
 156         .filter(|&(_, f)| f.level == Status::Unstable)
 157         .filter(|&(_, f)| !f.has_gate_test)
 158         .collect::<Vec<_>>();
 159
 160     for &(name, _) in gate_untested.iter() {
 161         println!("Expected a gate test for the feature '{name}'.");
 162         println!(
 163             "Hint: create a failing test file named 'feature-gate-{}.rs'\
 164                 \n      in the 'ui' test suite, with its failures due to\
 165                 \n      missing usage of `#![feature({})]`.",
 166             name, name
 167         );
 168         println!(
 169             "Hint: If you already have such a test and don't want to rename it,\
 170                 \n      you can also add a // gate-test-{} line to the test file.",
 171             name
 172         );
 173     }
 174
 175     if !gate_untested.is_empty() {
 176         tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
 177     }
 178
 179     let (version, channel) = get_version_and_channel(src_path);
 180
 181     let all_features_iter = features
 182         .iter()
 183         .map(|feat| (feat, "lang"))
 184         .chain(lib_features.iter().map(|feat| (feat, "lib")));
 185     for ((feature_name, feature), kind) in all_features_iter {
 186         let since = if let Some(since) = feature.since { since } else { continue };
 187         if since > version && since != Version::CurrentPlaceholder {
 188             tidy_error!(
 189                 bad,
 190                 "The stabilization version {since} of {kind} feature `{feature_name}` is newer than the current {version}"
 191             );
 192         }
 193         if channel == "nightly" && since == version {
 194             tidy_error!(
 195                 bad,
 196                 "The stabilization version {since} of {kind} feature `{feature_name}` is written out but should be {}",
 197                 version::VERSION_PLACEHOLDER
 198             );
 199         }
 200         if channel != "nightly" && since == Version::CurrentPlaceholder {
 201             tidy_error!(
 202                 bad,
 203                 "The placeholder use of {kind} feature `{feature_name}` is not allowed on the {channel} channel",
 204             );
 205         }
 206     }
 207
 208     if *bad {
 209         return CollectedFeatures { lib: lib_features, lang: features };
 210     }
 211
 212     if verbose {
 213         let mut lines = Vec::new();
 214         lines.extend(format_features(&features, "lang"));
 215         lines.extend(format_features(&lib_features, "lib"));
 216
 217         lines.sort();
 218         for line in lines {
 219             println!("* {line}");
 220         }
 221     } else {
 222         println!("* {} features", features.len());
 223     }
 224
 225     CollectedFeatures { lib: lib_features, lang: features }
 226 }
 227
 228 fn get_version_and_channel(src_path: &Path) -> (Version, String) {
 229     let version_str = t!(std::fs::read_to_string(src_path.join("version")));
 230     let version_str = version_str.trim();
 231     let version = t!(std::str::FromStr::from_str(&version_str).map_err(|e| format!("{e:?}")));
 232     let channel_str = t!(std::fs::read_to_string(src_path.join("ci").join("channel")));
 233     (version, channel_str.trim().to_owned())
 234 }
 235
 236 fn format_features<'a>(
 237     features: &'a Features,
 238     family: &'a str,
 239 ) -> impl Iterator<Item = String> + 'a {
 240     features.iter().map(move |(name, feature)| {
 241         format!(
 242             "{:<32} {:<8} {:<12} {:<8}",
 243             name,
 244             family,
 245             feature.level,
 246             feature.since.map_or("None".to_owned(), |since| since.to_string())
 247         )
 248     })
 249 }
 250
 251 fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
 252     lazy_static::lazy_static! {
 253         static ref ISSUE: Regex = Regex::new(r#"issue\s*=\s*"([^"]*)""#).unwrap();
 254         static ref FEATURE: Regex = Regex::new(r#"feature\s*=\s*"([^"]*)""#).unwrap();
 255         static ref SINCE: Regex = Regex::new(r#"since\s*=\s*"([^"]*)""#).unwrap();
 256     }
 257
 258     let r = match attr {
 259         "issue" => &*ISSUE,
 260         "feature" => &*FEATURE,
 261         "since" => &*SINCE,
 262         _ => unimplemented!("{attr} not handled"),
 263     };
 264
 265     r.captures(line).and_then(|c| c.get(1)).map(|m| m.as_str())
 266 }
 267
 268 fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
 269     let prefix = "feature_gate_";
 270     if filen_underscore.starts_with(prefix) {
 271         for (n, f) in features.iter_mut() {
 272             // Equivalent to filen_underscore == format!("feature_gate_{n}")
 273             if &filen_underscore[prefix.len()..] == n {
 274                 f.has_gate_test = true;
 275                 return true;
 276             }
 277         }
 278     }
 279     false
 280 }
 281
 282 pub fn collect_lang_features(base_compiler_path: &Path, bad: &mut bool) -> Features {
 283     let mut features = Features::new();
 284     collect_lang_features_in(&mut features, base_compiler_path, "active.rs", bad);
 285     collect_lang_features_in(&mut features, base_compiler_path, "accepted.rs", bad);
 286     collect_lang_features_in(&mut features, base_compiler_path, "removed.rs", bad);
 287     features
 288 }
 289
 290 fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, bad: &mut bool) {
 291     let path = base.join("rustc_feature").join("src").join(file);
 292     let contents = t!(fs::read_to_string(&path));
 293
 294     // We allow rustc-internal features to omit a tracking issue.
 295     // To make tidy accept omitting a tracking issue, group the list of features
 296     // without one inside `// no-tracking-issue` and `// no-tracking-issue-end`.
 297     let mut next_feature_omits_tracking_issue = false;
 298
 299     let mut in_feature_group = false;
 300     let mut prev_names = vec![];
 301
 302     let lines = contents.lines().zip(1..);
 303     for (line, line_number) in lines {
 304         let line = line.trim();
 305
 306         // Within -start and -end, the tracking issue can be omitted.
 307         match line {
 308             "// no-tracking-issue-start" => {
 309                 next_feature_omits_tracking_issue = true;
 310                 continue;
 311             }
 312             "// no-tracking-issue-end" => {
 313                 next_feature_omits_tracking_issue = false;
 314                 continue;
 315             }
 316             _ => {}
 317         }
 318
 319         if line.starts_with(FEATURE_GROUP_START_PREFIX) {
 320             if in_feature_group {
 321                 tidy_error!(
 322                     bad,
 323                     "{}:{}: \
 324                         new feature group is started without ending the previous one",
 325                     path.display(),
 326                     line_number,
 327                 );
 328             }
 329
 330             in_feature_group = true;
 331             prev_names = vec![];
 332             continue;
 333         } else if line.starts_with(FEATURE_GROUP_END_PREFIX) {
 334             in_feature_group = false;
 335             prev_names = vec![];
 336             continue;
 337         }
 338
 339         let mut parts = line.split(',');
 340         let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {
 341             Some("active") => Status::Unstable,
 342             Some("incomplete") => Status::Unstable,
 343             Some("removed") => Status::Removed,
 344             Some("accepted") => Status::Stable,
 345             _ => continue,
 346         };
 347         let name = parts.next().unwrap().trim();
 348
 349         let since_str = parts.next().unwrap().trim().trim_matches('"');
 350         let since = match since_str.parse() {
 351             Ok(since) => Some(since),
 352             Err(err) => {
 353                 tidy_error!(
 354                     bad,
 355                     "{}:{}: failed to parse since: {} ({:?})",
 356                     path.display(),
 357                     line_number,
 358                     since_str,
 359                     err,
 360                 );
 361                 None
 362             }
 363         };
 364         if in_feature_group {
 365             if prev_names.last() > Some(&name) {
 366                 // This assumes the user adds the feature name at the end of the list, as we're
 367                 // not looking ahead.
 368                 let correct_index = match prev_names.binary_search(&name) {
 369                     Ok(_) => {
 370                         // This only occurs when the feature name has already been declared.
 371                         tidy_error!(
 372                             bad,
 373                             "{}:{}: duplicate feature {}",
 374                             path.display(),
 375                             line_number,
 376                             name,
 377                         );
 378                         // skip any additional checks for this line
 379                         continue;
 380                     }
 381                     Err(index) => index,
 382                 };
 383
 384                 let correct_placement = if correct_index == 0 {
 385                     "at the beginning of the feature group".to_owned()
 386                 } else if correct_index == prev_names.len() {
 387                     // I don't believe this is reachable given the above assumption, but it
 388                     // doesn't hurt to be safe.
 389                     "at the end of the feature group".to_owned()
 390                 } else {
 391                     format!(
 392                         "between {} and {}",
 393                         prev_names[correct_index - 1],
 394                         prev_names[correct_index],
 395                     )
 396                 };
 397
 398                 tidy_error!(
 399                     bad,
 400                     "{}:{}: feature {} is not sorted by feature name (should be {})",
 401                     path.display(),
 402                     line_number,
 403                     name,
 404                     correct_placement,
 405                 );
 406             }
 407             prev_names.push(name);
 408         }
 409
 410         let issue_str = parts.next().unwrap().trim();
 411         let tracking_issue = if issue_str.starts_with("None") {
 412             if level == Status::Unstable && !next_feature_omits_tracking_issue {
 413                 tidy_error!(
 414                     bad,
 415                     "{}:{}: no tracking issue for feature {}",
 416                     path.display(),
 417                     line_number,
 418                     name,
 419                 );
 420             }
 421             None
 422         } else {
 423             let s = issue_str.split('(').nth(1).unwrap().split(')').next().unwrap();
 424             Some(s.parse().unwrap())
 425         };
 426         match features.entry(name.to_owned()) {
 427             Entry::Occupied(e) => {
 428                 tidy_error!(
 429                     bad,
 430                     "{}:{} feature {name} already specified with status '{}'",
 431                     path.display(),
 432                     line_number,
 433                     e.get().level,
 434                 );
 435             }
 436             Entry::Vacant(e) => {
 437                 e.insert(Feature { level, since, has_gate_test: false, tracking_issue });
 438             }
 439         }
 440     }
 441 }
 442
 443 fn get_and_check_lib_features(
 444     base_src_path: &Path,
 445     bad: &mut bool,
 446     lang_features: &Features,
 447 ) -> Features {
 448     let mut lib_features = Features::new();
 449     map_lib_features(base_src_path, &mut |res, file, line| match res {
 450         Ok((name, f)) => {
 451             let mut check_features = |f: &Feature, list: &Features, display: &str| {
 452                 if let Some(ref s) = list.get(name) {
 453                     if f.tracking_issue != s.tracking_issue && f.level != Status::Stable {
 454                         tidy_error!(
 455                             bad,
 456                             "{}:{}: `issue` \"{}\" mismatches the {} `issue` of \"{}\"",
 457                             file.display(),
 458                             line,
 459                             f.tracking_issue_display(),
 460                             display,
 461                             s.tracking_issue_display(),
 462                         );
 463                     }
 464                 }
 465             };
 466             check_features(&f, &lang_features, "corresponding lang feature");
 467             check_features(&f, &lib_features, "previous");
 468             lib_features.insert(name.to_owned(), f);
 469         }
 470         Err(msg) => {
 471             tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
 472         }
 473     });
 474     lib_features
 475 }
 476
 477 fn map_lib_features(
 478     base_src_path: &Path,
 479     mf: &mut dyn FnMut(Result<(&str, Feature), &str>, &Path, usize),
 480 ) {
 481     walk(
 482         base_src_path,
 483         &mut |path| filter_dirs(path) || path.ends_with("src/test"),
 484         &mut |entry, contents| {
 485             let file = entry.path();
 486             let filename = file.file_name().unwrap().to_string_lossy();
 487             if !filename.ends_with(".rs")
 488                 || filename == "features.rs"
 489                 || filename == "diagnostic_list.rs"
 490                 || filename == "error_codes.rs"
 491             {
 492                 return;
 493             }
 494
 495             // This is an early exit -- all the attributes we're concerned with must contain this:
 496             // * rustc_const_unstable(
 497             // * unstable(
 498             // * stable(
 499             if !contents.contains("stable(") {
 500                 return;
 501             }
 502
 503             let handle_issue_none = |s| match s {
 504                 "none" => None,
 505                 issue => {
 506                     let n = issue.parse().expect("issue number is not a valid integer");
 507                     assert_ne!(n, 0, "\"none\" should be used when there is no issue, not \"0\"");
 508                     NonZeroU32::new(n)
 509                 }
 510             };
 511             let mut becoming_feature: Option<(&str, Feature)> = None;
 512             let mut iter_lines = contents.lines().enumerate().peekable();
 513             while let Some((i, line)) = iter_lines.next() {
 514                 macro_rules! err {
 515                     ($msg:expr) => {{
 516                         mf(Err($msg), file, i + 1);
 517                         continue;
 518                     }};
 519                 }
 520
 521                 lazy_static::lazy_static! {
 522                     static ref COMMENT_LINE: Regex = Regex::new(r"^\s*//").unwrap();
 523                 }
 524                 // exclude commented out lines
 525                 if COMMENT_LINE.is_match(line) {
 526                     continue;
 527                 }
 528
 529                 if let Some((ref name, ref mut f)) = becoming_feature {
 530                     if f.tracking_issue.is_none() {
 531                         f.tracking_issue = find_attr_val(line, "issue").and_then(handle_issue_none);
 532                     }
 533                     if line.ends_with(']') {
 534                         mf(Ok((name, f.clone())), file, i + 1);
 535                     } else if !line.ends_with(',') && !line.ends_with('\\') && !line.ends_with('"')
 536                     {
 537                         // We need to bail here because we might have missed the
 538                         // end of a stability attribute above because the ']'
 539                         // might not have been at the end of the line.
 540                         // We could then get into the very unfortunate situation that
 541                         // we continue parsing the file assuming the current stability
 542                         // attribute has not ended, and ignoring possible feature
 543                         // attributes in the process.
 544                         err!("malformed stability attribute");
 545                     } else {
 546                         continue;
 547                     }
 548                 }
 549                 becoming_feature = None;
 550                 if line.contains("rustc_const_unstable(") {
 551                     // `const fn` features are handled specially.
 552                     let feature_name = match find_attr_val(line, "feature").or_else(|| {
 553                         iter_lines.peek().and_then(|next| find_attr_val(next.1, "feature"))
 554                     }) {
 555                         Some(name) => name,
 556                         None => err!("malformed stability attribute: missing `feature` key"),
 557                     };
 558                     let feature = Feature {
 559                         level: Status::Unstable,
 560                         since: None,
 561                         has_gate_test: false,
 562                         tracking_issue: find_attr_val(line, "issue").and_then(handle_issue_none),
 563                     };
 564                     mf(Ok((feature_name, feature)), file, i + 1);
 565                     continue;
 566                 }
 567                 let level = if line.contains("[unstable(") {
 568                     Status::Unstable
 569                 } else if line.contains("[stable(") {
 570                     Status::Stable
 571                 } else {
 572                     continue;
 573                 };
 574                 let feature_name = match find_attr_val(line, "feature")
 575                     .or_else(|| iter_lines.peek().and_then(|next| find_attr_val(next.1, "feature")))
 576                 {
 577                     Some(name) => name,
 578                     None => err!("malformed stability attribute: missing `feature` key"),
 579                 };
 580                 let since = match find_attr_val(line, "since").map(|x| x.parse()) {
 581                     Some(Ok(since)) => Some(since),
 582                     Some(Err(_err)) => {
 583                         err!("malformed stability attribute: can't parse `since` key");
 584                     }
 585                     None if level == Status::Stable => {
 586                         err!("malformed stability attribute: missing the `since` key");
 587                     }
 588                     None => None,
 589                 };
 590                 let tracking_issue = find_attr_val(line, "issue").and_then(handle_issue_none);
 591
 592                 let feature = Feature { level, since, has_gate_test: false, tracking_issue };
 593                 if line.contains(']') {
 594                     mf(Ok((feature_name, feature)), file, i + 1);
 595                 } else {
 596                     becoming_feature = Some((feature_name, feature));
 597                 }
 598             }
 599         },
 600     );
 601 }