src/tools/linkchecker/main.rs

   1 //! Script to check the validity of `href` links in our HTML documentation.
   2 //!
   3 //! In the past we've been quite error prone to writing in broken links as most
   4 //! of them are manually rather than automatically added. As files move over
   5 //! time or apis change old links become stale or broken. The purpose of this
   6 //! script is to check all relative links in our documentation to make sure they
   7 //! actually point to a valid place.
   8 //!
   9 //! Currently this doesn't actually do any HTML parsing or anything fancy like
  10 //! that, it just has a simple "regex" to search for `href` and `id` tags.
  11 //! These values are then translated to file URLs if possible and then the
  12 //! destination is asserted to exist.
  13 //!
  14 //! A few exceptions are allowed as there's known bugs in rustdoc, but this
  15 //! should catch the majority of "broken link" cases.
  16
  17 use std::cell::RefCell;
  18 use std::collections::{HashMap, HashSet};
  19 use std::env;
  20 use std::fs;
  21 use std::io::ErrorKind;
  22 use std::path::{Component, Path, PathBuf};
  23 use std::rc::Rc;
  24 use std::time::Instant;
  25
  26 use once_cell::sync::Lazy;
  27 use regex::Regex;
  28
  29 // Add linkcheck exceptions here
  30 // If at all possible you should use intra-doc links to avoid linkcheck issues. These
  31 // are cases where that does not work
  32 // [(generated_documentation_page, &[broken_links])]
  33 #[rustfmt::skip]
  34 const LINKCHECK_EXCEPTIONS: &[(&str, &[&str])] = &[
  35     // These try to link to std::collections, but are defined in alloc
  36     // https://github.com/rust-lang/rust/issues/74481
  37     ("std/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
  38     ("std/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
  39     ("alloc/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
  40     ("alloc/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
  41
  42     // These try to link to various things in std, but are defined in core.
  43     // The docs in std::primitive use proper intra-doc links, so these seem fine to special-case.
  44     // Most these are broken because liballoc uses `#[lang_item]` magic to define things on
  45     // primitives that aren't available in core.
  46     ("alloc/slice/trait.Join.html", &["#method.join"]),
  47     ("alloc/slice/trait.Concat.html", &["#method.concat"]),
  48     ("alloc/slice/index.html", &["#method.concat", "#method.join"]),
  49     ("alloc/vec/struct.Vec.html", &["#method.sort_by_key", "#method.sort_by_cached_key"]),
  50     ("core/primitive.str.html", &["#method.to_ascii_uppercase", "#method.to_ascii_lowercase"]),
  51     ("core/primitive.slice.html", &["#method.to_ascii_uppercase", "#method.to_ascii_lowercase",
  52                                     "core/slice::sort_by_key", "core\\slice::sort_by_key",
  53                                     "#method.sort_by_cached_key"]),
  54 ];
  55
  56 #[rustfmt::skip]
  57 const INTRA_DOC_LINK_EXCEPTIONS: &[(&str, &[&str])] = &[
  58     // This will never have links that are not in other pages.
  59     // To avoid repeating the exceptions twice, an empty list means all broken links are allowed.
  60     ("reference/print.html", &[]),
  61     // All the reference 'links' are actually ENBF highlighted as code
  62     ("reference/comments.html", &[
  63          "/</code> <code>!",
  64          "*</code> <code>!",
  65     ]),
  66     ("reference/identifiers.html", &[
  67          "a</code>-<code>z</code> <code>A</code>-<code>Z",
  68          "a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
  69          "a</code>-<code>z</code> <code>A</code>-<code>Z</code>] [<code>a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
  70     ]),
  71     ("reference/tokens.html", &[
  72          "0</code>-<code>1",
  73          "0</code>-<code>7",
  74          "0</code>-<code>9",
  75          "0</code>-<code>9",
  76          "0</code>-<code>9</code> <code>a</code>-<code>f</code> <code>A</code>-<code>F",
  77     ]),
  78     ("reference/notation.html", &[
  79          "b</code> <code>B",
  80          "a</code>-<code>z",
  81     ]),
  82     // This is being used in the sense of 'inclusive range', not a markdown link
  83     ("core/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
  84     ("std/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
  85     ("core/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
  86     ("alloc/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
  87     ("std/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
  88     ("core/primitive.str.html", &["begin</code>, <code>end"]),
  89     ("std/primitive.str.html", &["begin</code>, <code>end"]),
  90
  91 ];
  92
  93 static BROKEN_INTRA_DOC_LINK: Lazy<Regex> =
  94     Lazy::new(|| Regex::new(r#"\[<code>(.*)</code>\]"#).unwrap());
  95
  96 macro_rules! t {
  97     ($e:expr) => {
  98         match $e {
  99             Ok(e) => e,
 100             Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
 101         }
 102     };
 103 }
 104
 105 fn main() {
 106     let docs = env::args_os().nth(1).expect("doc path should be first argument");
 107     let docs = env::current_dir().unwrap().join(docs);
 108     let mut checker = Checker { root: docs.clone(), cache: HashMap::new() };
 109     let mut report = Report {
 110         errors: 0,
 111         start: Instant::now(),
 112         html_files: 0,
 113         html_redirects: 0,
 114         links_checked: 0,
 115         links_ignored_external: 0,
 116         links_ignored_exception: 0,
 117         intra_doc_exceptions: 0,
 118     };
 119     checker.walk(&docs, &mut report);
 120     report.report();
 121     if report.errors != 0 {
 122         println!("found some broken links");
 123         std::process::exit(1);
 124     }
 125 }
 126
 127 struct Checker {
 128     root: PathBuf,
 129     cache: Cache,
 130 }
 131
 132 struct Report {
 133     errors: u32,
 134     start: Instant,
 135     html_files: u32,
 136     html_redirects: u32,
 137     links_checked: u32,
 138     links_ignored_external: u32,
 139     links_ignored_exception: u32,
 140     intra_doc_exceptions: u32,
 141 }
 142
 143 /// A cache entry.
 144 enum FileEntry {
 145     /// An HTML file.
 146     ///
 147     /// This includes the contents of the HTML file, and an optional set of
 148     /// HTML IDs. The IDs are used for checking fragments. They are computed
 149     /// as-needed. The source is discarded (replaced with an empty string)
 150     /// after the file has been checked, to conserve on memory.
 151     HtmlFile { source: Rc<String>, ids: RefCell<HashSet<String>> },
 152     /// This file is an HTML redirect to the given local path.
 153     Redirect { target: PathBuf },
 154     /// This is not an HTML file.
 155     OtherFile,
 156     /// This is a directory.
 157     Dir,
 158     /// The file doesn't exist.
 159     Missing,
 160 }
 161
 162 /// A cache to speed up file access.
 163 type Cache = HashMap<String, FileEntry>;
 164
 165 fn small_url_encode(s: &str) -> String {
 166     s.replace("<", "%3C")
 167         .replace(">", "%3E")
 168         .replace(" ", "%20")
 169         .replace("?", "%3F")
 170         .replace("'", "%27")
 171         .replace("&", "%26")
 172         .replace(",", "%2C")
 173         .replace(":", "%3A")
 174         .replace(";", "%3B")
 175         .replace("[", "%5B")
 176         .replace("]", "%5D")
 177         .replace("\"", "%22")
 178 }
 179
 180 impl Checker {
 181     /// Primary entry point for walking the filesystem to find HTML files to check.
 182     fn walk(&mut self, dir: &Path, report: &mut Report) {
 183         for entry in t!(dir.read_dir()).map(|e| t!(e)) {
 184             let path = entry.path();
 185             // Goes through symlinks
 186             let metadata = t!(fs::metadata(&path));
 187             if metadata.is_dir() {
 188                 self.walk(&path, report);
 189             } else {
 190                 self.check(&path, report);
 191             }
 192         }
 193     }
 194
 195     /// Checks a single file.
 196     fn check(&mut self, file: &Path, report: &mut Report) {
 197         let (pretty_path, entry) = self.load_file(file, report);
 198         let source = match entry {
 199             FileEntry::Missing => panic!("missing file {:?} while walking", file),
 200             FileEntry::Dir => unreachable!("never with `check` path"),
 201             FileEntry::OtherFile => return,
 202             FileEntry::Redirect { .. } => return,
 203             FileEntry::HtmlFile { source, ids } => {
 204                 parse_ids(&mut ids.borrow_mut(), &pretty_path, source, report);
 205                 source.clone()
 206             }
 207         };
 208
 209         // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
 210         with_attrs_in_source(&source, " href", |url, i, base| {
 211             // Ignore external URLs
 212             if url.starts_with("http:")
 213                 || url.starts_with("https:")
 214                 || url.starts_with("javascript:")
 215                 || url.starts_with("ftp:")
 216                 || url.starts_with("irc:")
 217                 || url.starts_with("data:")
 218             {
 219                 report.links_ignored_external += 1;
 220                 return;
 221             }
 222             report.links_checked += 1;
 223             let (url, fragment) = match url.split_once('#') {
 224                 None => (url, None),
 225                 Some((url, fragment)) => (url, Some(fragment)),
 226             };
 227             // NB: the `splitn` always succeeds, even if the delimiter is not present.
 228             let url = url.splitn(2, '?').next().unwrap();
 229
 230             // Once we've plucked out the URL, parse it using our base url and
 231             // then try to extract a file path.
 232             let mut path = file.to_path_buf();
 233             if !base.is_empty() || !url.is_empty() {
 234                 path.pop();
 235                 for part in Path::new(base).join(url).components() {
 236                     match part {
 237                         Component::Prefix(_) | Component::RootDir => {
 238                             // Avoid absolute paths as they make the docs not
 239                             // relocatable by making assumptions on where the docs
 240                             // are hosted relative to the site root.
 241                             report.errors += 1;
 242                             println!(
 243                                 "{}:{}: absolute path - {}",
 244                                 pretty_path,
 245                                 i + 1,
 246                                 Path::new(base).join(url).display()
 247                             );
 248                             return;
 249                         }
 250                         Component::CurDir => {}
 251                         Component::ParentDir => {
 252                             path.pop();
 253                         }
 254                         Component::Normal(s) => {
 255                             path.push(s);
 256                         }
 257                     }
 258                 }
 259             }
 260
 261             let (target_pretty_path, target_entry) = self.load_file(&path, report);
 262             let (target_source, target_ids) = match target_entry {
 263                 FileEntry::Missing => {
 264                     if is_exception(file, &target_pretty_path) {
 265                         report.links_ignored_exception += 1;
 266                     } else {
 267                         report.errors += 1;
 268                         println!(
 269                             "{}:{}: broken link - `{}`",
 270                             pretty_path,
 271                             i + 1,
 272                             target_pretty_path
 273                         );
 274                     }
 275                     return;
 276                 }
 277                 FileEntry::Dir => {
 278                     // Links to directories show as directory listings when viewing
 279                     // the docs offline so it's best to avoid them.
 280                     report.errors += 1;
 281                     println!(
 282                         "{}:{}: directory link to `{}` \
 283                          (directory links should use index.html instead)",
 284                         pretty_path,
 285                         i + 1,
 286                         target_pretty_path
 287                     );
 288                     return;
 289                 }
 290                 FileEntry::OtherFile => return,
 291                 FileEntry::Redirect { target } => {
 292                     let t = target.clone();
 293                     drop(target);
 294                     let (target, redir_entry) = self.load_file(&t, report);
 295                     match redir_entry {
 296                         FileEntry::Missing => {
 297                             report.errors += 1;
 298                             println!(
 299                                 "{}:{}: broken redirect from `{}` to `{}`",
 300                                 pretty_path,
 301                                 i + 1,
 302                                 target_pretty_path,
 303                                 target
 304                             );
 305                             return;
 306                         }
 307                         FileEntry::Redirect { target } => {
 308                             // Redirect to a redirect, this link checker
 309                             // currently doesn't support this, since it would
 310                             // require cycle checking, etc.
 311                             report.errors += 1;
 312                             println!(
 313                                 "{}:{}: redirect from `{}` to `{}` \
 314                                  which is also a redirect (not supported)",
 315                                 pretty_path,
 316                                 i + 1,
 317                                 target_pretty_path,
 318                                 target.display()
 319                             );
 320                             return;
 321                         }
 322                         FileEntry::Dir => {
 323                             report.errors += 1;
 324                             println!(
 325                                 "{}:{}: redirect from `{}` to `{}` \
 326                                  which is a directory \
 327                                  (directory links should use index.html instead)",
 328                                 pretty_path,
 329                                 i + 1,
 330                                 target_pretty_path,
 331                                 target
 332                             );
 333                             return;
 334                         }
 335                         FileEntry::OtherFile => return,
 336                         FileEntry::HtmlFile { source, ids } => (source, ids),
 337                     }
 338                 }
 339                 FileEntry::HtmlFile { source, ids } => (source, ids),
 340             };
 341
 342             // Alright, if we've found an HTML file for the target link. If
 343             // this is a fragment link, also check that the `id` exists.
 344             if let Some(ref fragment) = fragment {
 345                 // Fragments like `#1-6` are most likely line numbers to be
 346                 // interpreted by javascript, so we're ignoring these
 347                 if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
 348                     return;
 349                 }
 350
 351                 // These appear to be broken in mdbook right now?
 352                 if fragment.starts_with('-') {
 353                     return;
 354                 }
 355
 356                 parse_ids(&mut target_ids.borrow_mut(), &pretty_path, target_source, report);
 357
 358                 if target_ids.borrow().contains(*fragment) {
 359                     return;
 360                 }
 361
 362                 if is_exception(file, &format!("#{}", fragment)) {
 363                     report.links_ignored_exception += 1;
 364                 } else {
 365                     report.errors += 1;
 366                     print!("{}:{}: broken link fragment ", pretty_path, i + 1);
 367                     println!("`#{}` pointing to `{}`", fragment, target_pretty_path);
 368                 };
 369             }
 370         });
 371
 372         // Search for intra-doc links that rustdoc didn't warn about
 373         // FIXME(#77199, 77200) Rustdoc should just warn about these directly.
 374         // NOTE: only looks at one line at a time; in practice this should find most links
 375         for (i, line) in source.lines().enumerate() {
 376             for broken_link in BROKEN_INTRA_DOC_LINK.captures_iter(line) {
 377                 if is_intra_doc_exception(file, &broken_link[1]) {
 378                     report.intra_doc_exceptions += 1;
 379                 } else {
 380                     report.errors += 1;
 381                     print!("{}:{}: broken intra-doc link - ", pretty_path, i + 1);
 382                     println!("{}", &broken_link[0]);
 383                 }
 384             }
 385         }
 386         // we don't need the source anymore,
 387         // so drop to reduce memory-usage
 388         match self.cache.get_mut(&pretty_path).unwrap() {
 389             FileEntry::HtmlFile { source, .. } => *source = Rc::new(String::new()),
 390             _ => unreachable!("must be html file"),
 391         }
 392     }
 393
 394     /// Load a file from disk, or from the cache if available.
 395     fn load_file(&mut self, file: &Path, report: &mut Report) -> (String, &FileEntry) {
 396         // https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
 397         #[cfg(windows)]
 398         const ERROR_INVALID_NAME: i32 = 123;
 399
 400         let pretty_path =
 401             file.strip_prefix(&self.root).unwrap_or(&file).to_str().unwrap().to_string();
 402
 403         let entry =
 404             self.cache.entry(pretty_path.clone()).or_insert_with(|| match fs::metadata(file) {
 405                 Ok(metadata) if metadata.is_dir() => FileEntry::Dir,
 406                 Ok(_) => {
 407                     if file.extension().and_then(|s| s.to_str()) != Some("html") {
 408                         FileEntry::OtherFile
 409                     } else {
 410                         report.html_files += 1;
 411                         load_html_file(file, report)
 412                     }
 413                 }
 414                 Err(e) if e.kind() == ErrorKind::NotFound => FileEntry::Missing,
 415                 Err(e) => {
 416                     // If a broken intra-doc link contains `::`, on windows, it will cause `ERROR_INVALID_NAME` rather than `NotFound`.
 417                     // Explicitly check for that so that the broken link can be allowed in `LINKCHECK_EXCEPTIONS`.
 418                     #[cfg(windows)]
 419                     if e.raw_os_error() == Some(ERROR_INVALID_NAME)
 420                         && file.as_os_str().to_str().map_or(false, |s| s.contains("::"))
 421                     {
 422                         return FileEntry::Missing;
 423                     }
 424                     panic!("unexpected read error for {}: {}", file.display(), e);
 425                 }
 426             });
 427         (pretty_path, entry)
 428     }
 429 }
 430
 431 impl Report {
 432     fn report(&self) {
 433         println!("checked links in: {:.1}s", self.start.elapsed().as_secs_f64());
 434         println!("number of HTML files scanned: {}", self.html_files);
 435         println!("number of HTML redirects found: {}", self.html_redirects);
 436         println!("number of links checked: {}", self.links_checked);
 437         println!("number of links ignored due to external: {}", self.links_ignored_external);
 438         println!("number of links ignored due to exceptions: {}", self.links_ignored_exception);
 439         println!("number of intra doc links ignored: {}", self.intra_doc_exceptions);
 440         println!("errors found: {}", self.errors);
 441     }
 442 }
 443
 444 fn load_html_file(file: &Path, report: &mut Report) -> FileEntry {
 445     let source = match fs::read_to_string(file) {
 446         Ok(s) => Rc::new(s),
 447         Err(err) => {
 448             // This usually should not fail since `metadata` was already
 449             // called successfully on this file.
 450             panic!("unexpected read error for {}: {}", file.display(), err);
 451         }
 452     };
 453     match maybe_redirect(&source) {
 454         Some(target) => {
 455             report.html_redirects += 1;
 456             let target = file.parent().unwrap().join(target);
 457             FileEntry::Redirect { target }
 458         }
 459         None => FileEntry::HtmlFile { source: source.clone(), ids: RefCell::new(HashSet::new()) },
 460     }
 461 }
 462
 463 fn is_intra_doc_exception(file: &Path, link: &str) -> bool {
 464     if let Some(entry) = INTRA_DOC_LINK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
 465         entry.1.is_empty() || entry.1.contains(&link)
 466     } else {
 467         false
 468     }
 469 }
 470
 471 fn is_exception(file: &Path, link: &str) -> bool {
 472     if let Some(entry) = LINKCHECK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
 473         entry.1.contains(&link)
 474     } else {
 475         // FIXME(#63351): Concat trait in alloc/slice reexported in primitive page
 476         //
 477         // NOTE: This cannot be added to `LINKCHECK_EXCEPTIONS` because the resolved path
 478         // calculated in `check` function is outside `build/<triple>/doc` dir.
 479         // So the `strip_prefix` method just returns the old absolute broken path.
 480         if file.ends_with("std/primitive.slice.html") {
 481             if link.ends_with("primitive.slice.html") {
 482                 return true;
 483             }
 484         }
 485         false
 486     }
 487 }
 488
 489 /// If the given HTML file contents is an HTML redirect, this returns the
 490 /// destination path given in the redirect.
 491 fn maybe_redirect(source: &str) -> Option<String> {
 492     const REDIRECT_RUSTDOC: (usize, &str) = (7, "<p>Redirecting to <a href=");
 493     const REDIRECT_MDBOOK: (usize, &str) = (8 - 7, "<p>Redirecting to... <a href=");
 494
 495     let mut lines = source.lines();
 496
 497     let mut find_redirect = |(line_rel, redirect_pattern): (usize, &str)| {
 498         let redirect_line = lines.nth(line_rel)?;
 499
 500         redirect_line.find(redirect_pattern).map(|i| {
 501             let rest = &redirect_line[(i + redirect_pattern.len() + 1)..];
 502             let pos_quote = rest.find('"').unwrap();
 503             rest[..pos_quote].to_owned()
 504         })
 505     };
 506
 507     find_redirect(REDIRECT_RUSTDOC).or_else(|| find_redirect(REDIRECT_MDBOOK))
 508 }
 509
 510 fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(source: &str, attr: &str, mut f: F) {
 511     let mut base = "";
 512     for (i, mut line) in source.lines().enumerate() {
 513         while let Some(j) = line.find(attr) {
 514             let rest = &line[j + attr.len()..];
 515             // The base tag should always be the first link in the document so
 516             // we can get away with using one pass.
 517             let is_base = line[..j].ends_with("<base");
 518             line = rest;
 519             let pos_equals = match rest.find('=') {
 520                 Some(i) => i,
 521                 None => continue,
 522             };
 523             if rest[..pos_equals].trim_start_matches(' ') != "" {
 524                 continue;
 525             }
 526
 527             let rest = &rest[pos_equals + 1..];
 528
 529             let pos_quote = match rest.find(&['"', '\''][..]) {
 530                 Some(i) => i,
 531                 None => continue,
 532             };
 533             let quote_delim = rest.as_bytes()[pos_quote] as char;
 534
 535             if rest[..pos_quote].trim_start_matches(' ') != "" {
 536                 continue;
 537             }
 538             let rest = &rest[pos_quote + 1..];
 539             let url = match rest.find(quote_delim) {
 540                 Some(i) => &rest[..i],
 541                 None => continue,
 542             };
 543             if is_base {
 544                 base = url;
 545                 continue;
 546             }
 547             f(url, i, base)
 548         }
 549     }
 550 }
 551
 552 fn parse_ids(ids: &mut HashSet<String>, file: &str, source: &str, report: &mut Report) {
 553     if ids.is_empty() {
 554         with_attrs_in_source(source, " id", |fragment, i, _| {
 555             let frag = fragment.trim_start_matches("#").to_owned();
 556             let encoded = small_url_encode(&frag);
 557             if !ids.insert(frag) {
 558                 report.errors += 1;
 559                 println!("{}:{}: id is not unique: `{}`", file, i, fragment);
 560             }
 561             // Just in case, we also add the encoded id.
 562             ids.insert(encoded);
 563         });
 564     }
 565 }