1 //! Script to check the validity of `href` links in our HTML documentation.
3 //! In the past we've been quite error prone to writing in broken links as most
4 //! of them are manually rather than automatically added. As files move over
5 //! time or apis change old links become stale or broken. The purpose of this
6 //! script is to check all relative links in our documentation to make sure they
7 //! actually point to a valid place.
9 //! Currently this doesn't actually do any HTML parsing or anything fancy like
10 //! that, it just has a simple "regex" to search for `href` and `id` tags.
11 //! These values are then translated to file URLs if possible and then the
12 //! destination is asserted to exist.
14 //! A few exceptions are allowed as there's known bugs in rustdoc, but this
15 //! should catch the majority of "broken link" cases.
17 use std::cell::RefCell;
18 use std::collections::{HashMap, HashSet};
21 use std::io::ErrorKind;
22 use std::path::{Component, Path, PathBuf};
24 use std::time::Instant;
26 use once_cell::sync::Lazy;
29 // Add linkcheck exceptions here
30 // If at all possible you should use intra-doc links to avoid linkcheck issues. These
31 // are cases where that does not work
32 // [(generated_documentation_page, &[broken_links])]
34 const LINKCHECK_EXCEPTIONS: &[(&str, &[&str])] = &[
35 // These try to link to std::collections, but are defined in alloc
36 // https://github.com/rust-lang/rust/issues/74481
37 ("std/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
38 ("std/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
39 ("alloc/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
40 ("alloc/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
42 // These try to link to various things in std, but are defined in core.
43 // The docs in std::primitive use proper intra-doc links, so these seem fine to special-case.
44 // Most these are broken because liballoc uses `#[lang_item]` magic to define things on
45 // primitives that aren't available in core.
46 ("alloc/slice/trait.Join.html", &["#method.join"]),
47 ("alloc/slice/trait.Concat.html", &["#method.concat"]),
48 ("alloc/slice/index.html", &["#method.concat", "#method.join"]),
49 ("alloc/vec/struct.Vec.html", &["#method.sort_by_key", "#method.sort_by_cached_key"]),
50 ("core/primitive.str.html", &["#method.to_ascii_uppercase", "#method.to_ascii_lowercase"]),
51 ("core/primitive.slice.html", &["#method.to_ascii_uppercase", "#method.to_ascii_lowercase",
52 "core/slice::sort_by_key", "core\\slice::sort_by_key",
53 "#method.sort_by_cached_key"]),
57 const INTRA_DOC_LINK_EXCEPTIONS: &[(&str, &[&str])] = &[
58 // This will never have links that are not in other pages.
59 // To avoid repeating the exceptions twice, an empty list means all broken links are allowed.
60 ("reference/print.html", &[]),
61 // All the reference 'links' are actually ENBF highlighted as code
62 ("reference/comments.html", &[
66 ("reference/identifiers.html", &[
67 "a</code>-<code>z</code> <code>A</code>-<code>Z",
68 "a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
69 "a</code>-<code>z</code> <code>A</code>-<code>Z</code>] [<code>a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
71 ("reference/tokens.html", &[
76 "0</code>-<code>9</code> <code>a</code>-<code>f</code> <code>A</code>-<code>F",
78 ("reference/notation.html", &[
82 // This is being used in the sense of 'inclusive range', not a markdown link
83 ("core/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
84 ("std/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
85 ("core/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
86 ("alloc/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
87 ("std/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
88 ("core/primitive.str.html", &["begin</code>, <code>end"]),
89 ("std/primitive.str.html", &["begin</code>, <code>end"]),
93 static BROKEN_INTRA_DOC_LINK: Lazy<Regex> =
94 Lazy::new(|| Regex::new(r#"\[<code>(.*)</code>\]"#).unwrap());
100 Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
106 let docs = env::args_os().nth(1).expect("doc path should be first argument");
107 let docs = env::current_dir().unwrap().join(docs);
108 let mut checker = Checker { root: docs.clone(), cache: HashMap::new() };
109 let mut report = Report {
111 start: Instant::now(),
115 links_ignored_external: 0,
116 links_ignored_exception: 0,
117 intra_doc_exceptions: 0,
119 checker.walk(&docs, &mut report);
121 if report.errors != 0 {
122 println!("found some broken links");
123 std::process::exit(1);
138 links_ignored_external: u32,
139 links_ignored_exception: u32,
140 intra_doc_exceptions: u32,
147 /// This includes the contents of the HTML file, and an optional set of
148 /// HTML IDs. The IDs are used for checking fragments. They are computed
149 /// as-needed. The source is discarded (replaced with an empty string)
150 /// after the file has been checked, to conserve on memory.
151 HtmlFile { source: Rc<String>, ids: RefCell<HashSet<String>> },
152 /// This file is an HTML redirect to the given local path.
153 Redirect { target: PathBuf },
154 /// This is not an HTML file.
156 /// This is a directory.
158 /// The file doesn't exist.
162 /// A cache to speed up file access.
163 type Cache = HashMap<String, FileEntry>;
165 fn small_url_encode(s: &str) -> String {
166 s.replace("<", "%3C")
177 .replace("\"", "%22")
181 /// Primary entry point for walking the filesystem to find HTML files to check.
182 fn walk(&mut self, dir: &Path, report: &mut Report) {
183 for entry in t!(dir.read_dir()).map(|e| t!(e)) {
184 let path = entry.path();
185 let kind = t!(entry.file_type());
187 self.walk(&path, report);
189 self.check(&path, report);
194 /// Checks a single file.
195 fn check(&mut self, file: &Path, report: &mut Report) {
196 let (pretty_path, entry) = self.load_file(file, report);
197 let source = match entry {
198 FileEntry::Missing => panic!("missing file {:?} while walking", file),
199 FileEntry::Dir => unreachable!("never with `check` path"),
200 FileEntry::OtherFile => return,
201 FileEntry::Redirect { .. } => return,
202 FileEntry::HtmlFile { source, ids } => {
203 parse_ids(&mut ids.borrow_mut(), &pretty_path, source, report);
208 // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
209 with_attrs_in_source(&source, " href", |url, i, base| {
210 // Ignore external URLs
211 if url.starts_with("http:")
212 || url.starts_with("https:")
213 || url.starts_with("javascript:")
214 || url.starts_with("ftp:")
215 || url.starts_with("irc:")
216 || url.starts_with("data:")
218 report.links_ignored_external += 1;
221 report.links_checked += 1;
222 let (url, fragment) = match url.split_once('#') {
224 Some((url, fragment)) => (url, Some(fragment)),
226 // NB: the `splitn` always succeeds, even if the delimiter is not present.
227 let url = url.splitn(2, '?').next().unwrap();
229 // Once we've plucked out the URL, parse it using our base url and
230 // then try to extract a file path.
231 let mut path = file.to_path_buf();
232 if !base.is_empty() || !url.is_empty() {
234 for part in Path::new(base).join(url).components() {
236 Component::Prefix(_) | Component::RootDir => {
237 // Avoid absolute paths as they make the docs not
238 // relocatable by making assumptions on where the docs
239 // are hosted relative to the site root.
242 "{}:{}: absolute path - {}",
245 Path::new(base).join(url).display()
249 Component::CurDir => {}
250 Component::ParentDir => {
253 Component::Normal(s) => {
260 let (target_pretty_path, target_entry) = self.load_file(&path, report);
261 let (target_source, target_ids) = match target_entry {
262 FileEntry::Missing => {
263 if is_exception(file, &target_pretty_path) {
264 report.links_ignored_exception += 1;
268 "{}:{}: broken link - `{}`",
277 // Links to directories show as directory listings when viewing
278 // the docs offline so it's best to avoid them.
281 "{}:{}: directory link to `{}` \
282 (directory links should use index.html instead)",
289 FileEntry::OtherFile => return,
290 FileEntry::Redirect { target } => {
291 let t = target.clone();
293 let (target, redir_entry) = self.load_file(&t, report);
295 FileEntry::Missing => {
298 "{}:{}: broken redirect from `{}` to `{}`",
306 FileEntry::Redirect { target } => {
307 // Redirect to a redirect, this link checker
308 // currently doesn't support this, since it would
309 // require cycle checking, etc.
312 "{}:{}: redirect from `{}` to `{}` \
313 which is also a redirect (not supported)",
324 "{}:{}: redirect from `{}` to `{}` \
325 which is a directory \
326 (directory links should use index.html instead)",
334 FileEntry::OtherFile => return,
335 FileEntry::HtmlFile { source, ids } => (source, ids),
338 FileEntry::HtmlFile { source, ids } => (source, ids),
341 // Alright, if we've found an HTML file for the target link. If
342 // this is a fragment link, also check that the `id` exists.
343 if let Some(ref fragment) = fragment {
344 // Fragments like `#1-6` are most likely line numbers to be
345 // interpreted by javascript, so we're ignoring these
346 if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
350 // These appear to be broken in mdbook right now?
351 if fragment.starts_with('-') {
355 parse_ids(&mut target_ids.borrow_mut(), &pretty_path, target_source, report);
357 if target_ids.borrow().contains(*fragment) {
361 if is_exception(file, &format!("#{}", fragment)) {
362 report.links_ignored_exception += 1;
365 print!("{}:{}: broken link fragment ", pretty_path, i + 1);
366 println!("`#{}` pointing to `{}`", fragment, target_pretty_path);
371 // Search for intra-doc links that rustdoc didn't warn about
372 // FIXME(#77199, 77200) Rustdoc should just warn about these directly.
373 // NOTE: only looks at one line at a time; in practice this should find most links
374 for (i, line) in source.lines().enumerate() {
375 for broken_link in BROKEN_INTRA_DOC_LINK.captures_iter(line) {
376 if is_intra_doc_exception(file, &broken_link[1]) {
377 report.intra_doc_exceptions += 1;
380 print!("{}:{}: broken intra-doc link - ", pretty_path, i + 1);
381 println!("{}", &broken_link[0]);
385 // we don't need the source anymore,
386 // so drop to reduce memory-usage
387 match self.cache.get_mut(&pretty_path).unwrap() {
388 FileEntry::HtmlFile { source, .. } => *source = Rc::new(String::new()),
389 _ => unreachable!("must be html file"),
393 /// Load a file from disk, or from the cache if available.
394 fn load_file(&mut self, file: &Path, report: &mut Report) -> (String, &FileEntry) {
395 // https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
397 const ERROR_INVALID_NAME: i32 = 123;
400 file.strip_prefix(&self.root).unwrap_or(&file).to_str().unwrap().to_string();
403 self.cache.entry(pretty_path.clone()).or_insert_with(|| match fs::metadata(file) {
404 Ok(metadata) if metadata.is_dir() => FileEntry::Dir,
406 if file.extension().and_then(|s| s.to_str()) != Some("html") {
409 report.html_files += 1;
410 load_html_file(file, report)
413 Err(e) if e.kind() == ErrorKind::NotFound => FileEntry::Missing,
415 // If a broken intra-doc link contains `::`, on windows, it will cause `ERROR_INVALID_NAME` rather than `NotFound`.
416 // Explicitly check for that so that the broken link can be allowed in `LINKCHECK_EXCEPTIONS`.
418 if e.raw_os_error() == Some(ERROR_INVALID_NAME)
419 && file.as_os_str().to_str().map_or(false, |s| s.contains("::"))
421 return FileEntry::Missing;
423 panic!("unexpected read error for {}: {}", file.display(), e);
432 println!("checked links in: {:.1}s", self.start.elapsed().as_secs_f64());
433 println!("number of HTML files scanned: {}", self.html_files);
434 println!("number of HTML redirects found: {}", self.html_redirects);
435 println!("number of links checked: {}", self.links_checked);
436 println!("number of links ignored due to external: {}", self.links_ignored_external);
437 println!("number of links ignored due to exceptions: {}", self.links_ignored_exception);
438 println!("number of intra doc links ignored: {}", self.intra_doc_exceptions);
439 println!("errors found: {}", self.errors);
443 fn load_html_file(file: &Path, report: &mut Report) -> FileEntry {
444 let source = match fs::read_to_string(file) {
447 // This usually should not fail since `metadata` was already
448 // called successfully on this file.
449 panic!("unexpected read error for {}: {}", file.display(), err);
452 match maybe_redirect(&source) {
454 report.html_redirects += 1;
455 let target = file.parent().unwrap().join(target);
456 FileEntry::Redirect { target }
458 None => FileEntry::HtmlFile { source: source.clone(), ids: RefCell::new(HashSet::new()) },
462 fn is_intra_doc_exception(file: &Path, link: &str) -> bool {
463 if let Some(entry) = INTRA_DOC_LINK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
464 entry.1.is_empty() || entry.1.contains(&link)
470 fn is_exception(file: &Path, link: &str) -> bool {
471 if let Some(entry) = LINKCHECK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
472 entry.1.contains(&link)
474 // FIXME(#63351): Concat trait in alloc/slice reexported in primitive page
476 // NOTE: This cannot be added to `LINKCHECK_EXCEPTIONS` because the resolved path
477 // calculated in `check` function is outside `build/<triple>/doc` dir.
478 // So the `strip_prefix` method just returns the old absolute broken path.
479 if file.ends_with("std/primitive.slice.html") {
480 if link.ends_with("primitive.slice.html") {
488 /// If the given HTML file contents is an HTML redirect, this returns the
489 /// destination path given in the redirect.
490 fn maybe_redirect(source: &str) -> Option<String> {
491 const REDIRECT: &str = "<p>Redirecting to <a href=";
493 let mut lines = source.lines();
494 let redirect_line = lines.nth(7)?;
496 redirect_line.find(REDIRECT).map(|i| {
497 let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
498 let pos_quote = rest.find('"').unwrap();
499 rest[..pos_quote].to_owned()
503 fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(source: &str, attr: &str, mut f: F) {
505 for (i, mut line) in source.lines().enumerate() {
506 while let Some(j) = line.find(attr) {
507 let rest = &line[j + attr.len()..];
508 // The base tag should always be the first link in the document so
509 // we can get away with using one pass.
510 let is_base = line[..j].ends_with("<base");
512 let pos_equals = match rest.find('=') {
516 if rest[..pos_equals].trim_start_matches(' ') != "" {
520 let rest = &rest[pos_equals + 1..];
522 let pos_quote = match rest.find(&['"', '\''][..]) {
526 let quote_delim = rest.as_bytes()[pos_quote] as char;
528 if rest[..pos_quote].trim_start_matches(' ') != "" {
531 let rest = &rest[pos_quote + 1..];
532 let url = match rest.find(quote_delim) {
533 Some(i) => &rest[..i],
545 fn parse_ids(ids: &mut HashSet<String>, file: &str, source: &str, report: &mut Report) {
547 with_attrs_in_source(source, " id", |fragment, i, _| {
548 let frag = fragment.trim_start_matches("#").to_owned();
549 let encoded = small_url_encode(&frag);
550 if !ids.insert(frag) {
552 println!("{}:{}: id is not unique: `{}`", file, i, fragment);
554 // Just in case, we also add the encoded id.