1 //! Script to check the validity of `href` links in our HTML documentation.
3 //! In the past we've been quite error prone to writing in broken links as most
4 //! of them are manually rather than automatically added. As files move over
5 //! time or apis change old links become stale or broken. The purpose of this
6 //! script is to check all relative links in our documentation to make sure they
7 //! actually point to a valid place.
9 //! Currently this doesn't actually do any HTML parsing or anything fancy like
10 //! that, it just has a simple "regex" to search for `href` and `id` tags.
11 //! These values are then translated to file URLs if possible and then the
12 //! destination is asserted to exist.
14 //! A few exceptions are allowed as there's known bugs in rustdoc, but this
15 //! should catch the majority of "broken link" cases.
17 use std::collections::hash_map::Entry;
18 use std::collections::{HashMap, HashSet};
21 use std::path::{Component, Path, PathBuf};
24 use once_cell::sync::Lazy;
27 use crate::Redirect::*;
29 // Add linkcheck exceptions here
30 // If at all possible you should use intra-doc links to avoid linkcheck issues. These
31 // are cases where that does not work
32 // [(generated_documentation_page, &[broken_links])]
33 const LINKCHECK_EXCEPTIONS: &[(&str, &[&str])] = &[
34 // These try to link to std::collections, but are defined in alloc
35 // https://github.com/rust-lang/rust/issues/74481
36 ("std/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
37 ("std/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
38 ("alloc/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
39 ("alloc/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
43 const INTRA_DOC_LINK_EXCEPTIONS: &[(&str, &[&str])] = &[
44 // This will never have links that are not in other pages.
45 // To avoid repeating the exceptions twice, an empty list means all broken links are allowed.
46 ("reference/print.html", &[]),
47 // All the reference 'links' are actually ENBF highlighted as code
48 ("reference/comments.html", &[
52 ("reference/identifiers.html", &[
53 "a</code>-<code>z</code> <code>A</code>-<code>Z",
54 "a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
55 "a</code>-<code>z</code> <code>A</code>-<code>Z</code>] [<code>a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
57 ("reference/tokens.html", &[
62 "0</code>-<code>9</code> <code>a</code>-<code>f</code> <code>A</code>-<code>F",
64 ("reference/notation.html", &[
68 // This is being used in the sense of 'inclusive range', not a markdown link
69 ("core/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
70 ("std/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
71 ("core/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
72 ("alloc/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
73 ("std/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
77 static BROKEN_INTRA_DOC_LINK: Lazy<Regex> =
78 Lazy::new(|| Regex::new(r#"\[<code>(.*)</code>\]"#).unwrap());
84 Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
90 let docs = env::args_os().nth(1).unwrap();
91 let docs = env::current_dir().unwrap().join(docs);
92 let mut errors = false;
93 walk(&mut HashMap::new(), &docs, &docs, &mut errors);
95 panic!("found some broken links");
101 IOError(std::io::Error),
102 BrokenRedirect(PathBuf, std::io::Error),
113 ids: HashSet<String>,
116 type Cache = HashMap<PathBuf, FileEntry>;
118 fn small_url_encode(s: &str) -> String {
119 s.replace("<", "%3C")
130 .replace("\"", "%22")
134 fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
135 if self.ids.is_empty() {
136 with_attrs_in_source(contents, " id", |fragment, i, _| {
137 let frag = fragment.trim_start_matches("#").to_owned();
138 let encoded = small_url_encode(&frag);
139 if !self.ids.insert(frag) {
141 println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
143 // Just in case, we also add the encoded id.
144 self.ids.insert(encoded);
150 fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
151 for entry in t!(dir.read_dir()).map(|e| t!(e)) {
152 let path = entry.path();
153 let kind = t!(entry.file_type());
155 walk(cache, root, &path, errors);
157 let pretty_path = check(cache, root, &path, errors);
158 if let Some(pretty_path) = pretty_path {
159 let entry = cache.get_mut(&pretty_path).unwrap();
160 // we don't need the source anymore,
161 // so drop to reduce memory-usage
162 entry.source = Rc::new(String::new());
168 fn is_intra_doc_exception(file: &Path, link: &str) -> bool {
169 if let Some(entry) = INTRA_DOC_LINK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
170 entry.1.is_empty() || entry.1.contains(&link)
176 fn is_exception(file: &Path, link: &str) -> bool {
177 if let Some(entry) = LINKCHECK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
178 entry.1.contains(&link)
180 // FIXME(#63351): Concat trait in alloc/slice reexported in primitive page
182 // NOTE: This cannot be added to `LINKCHECK_EXCEPTIONS` because the resolved path
183 // calculated in `check` function is outside `build/<triple>/doc` dir.
184 // So the `strip_prefix` method just returns the old absolute broken path.
185 if file.ends_with("std/primitive.slice.html") {
186 if link.ends_with("primitive.slice.html") {
194 fn check(cache: &mut Cache, root: &Path, file: &Path, errors: &mut bool) -> Option<PathBuf> {
195 // Ignore non-HTML files.
196 if file.extension().and_then(|s| s.to_str()) != Some("html") {
200 let res = load_file(cache, root, file, SkipRedirect);
201 let (pretty_file, contents) = match res {
203 Err(_) => return None,
206 cache.get_mut(&pretty_file).unwrap().parse_ids(&pretty_file, &contents, errors);
209 // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
210 with_attrs_in_source(&contents, " href", |url, i, base| {
211 // Ignore external URLs
212 if url.starts_with("http:")
213 || url.starts_with("https:")
214 || url.starts_with("javascript:")
215 || url.starts_with("ftp:")
216 || url.starts_with("irc:")
217 || url.starts_with("data:")
221 let (url, fragment) = match url.split_once('#') {
223 Some((url, fragment)) => (url, Some(fragment)),
225 // NB: the `splitn` always succeeds, even if the delimiter is not present.
226 let url = url.splitn(2, '?').next().unwrap();
228 // Once we've plucked out the URL, parse it using our base url and
229 // then try to extract a file path.
230 let mut path = file.to_path_buf();
231 if !base.is_empty() || !url.is_empty() {
233 for part in Path::new(base).join(url).components() {
235 Component::Prefix(_) | Component::RootDir => {
236 // Avoid absolute paths as they make the docs not
237 // relocatable by making assumptions on where the docs
238 // are hosted relative to the site root.
241 "{}:{}: absolute path - {}",
242 pretty_file.display(),
244 Path::new(base).join(url).display()
248 Component::CurDir => {}
249 Component::ParentDir => {
252 Component::Normal(s) => {
259 // Alright, if we've found a file name then this file had better
260 // exist! If it doesn't then we register and print an error.
263 // Links to directories show as directory listings when viewing
264 // the docs offline so it's best to avoid them.
266 let pretty_path = path.strip_prefix(root).unwrap_or(&path);
268 "{}:{}: directory link - {}",
269 pretty_file.display(),
271 pretty_path.display()
275 if let Some(extension) = path.extension() {
276 // Ignore none HTML files.
277 if extension != "html" {
281 let res = load_file(cache, root, &path, FromRedirect(false));
282 let (pretty_path, contents) = match res {
284 Err(LoadError::IOError(err)) => {
285 panic!("error loading {}: {}", path.display(), err);
287 Err(LoadError::BrokenRedirect(target, _)) => {
290 "{}:{}: broken redirect to {}",
291 pretty_file.display(),
297 Err(LoadError::IsRedirect) => unreachable!(),
300 if let Some(ref fragment) = fragment {
301 // Fragments like `#1-6` are most likely line numbers to be
302 // interpreted by javascript, so we're ignoring these
303 if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
307 // These appear to be broken in mdbook right now?
308 if fragment.starts_with('-') {
312 let entry = &mut cache.get_mut(&pretty_path).unwrap();
313 entry.parse_ids(&pretty_path, &contents, errors);
315 if !entry.ids.contains(*fragment) && !is_exception(file, &format!("#{}", fragment))
318 print!("{}:{}: broken link fragment ", pretty_file.display(), i + 1);
319 println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
323 let pretty_path = path.strip_prefix(root).unwrap_or(&path);
324 if !is_exception(file, pretty_path.to_str().unwrap()) {
326 print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
327 println!("{}", pretty_path.display());
332 // Search for intra-doc links that rustdoc didn't warn about
333 // FIXME(#77199, 77200) Rustdoc should just warn about these directly.
334 // NOTE: only looks at one line at a time; in practice this should find most links
335 for (i, line) in contents.lines().enumerate() {
336 for broken_link in BROKEN_INTRA_DOC_LINK.captures_iter(line) {
337 if !is_intra_doc_exception(file, &broken_link[1]) {
339 print!("{}:{}: broken intra-doc link - ", pretty_file.display(), i + 1);
340 println!("{}", &broken_link[0]);
352 ) -> Result<(PathBuf, Rc<String>), LoadError> {
353 let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));
355 let (maybe_redirect, contents) = match cache.entry(pretty_file.clone()) {
356 Entry::Occupied(entry) => (None, entry.get().source.clone()),
357 Entry::Vacant(entry) => {
358 let contents = match fs::read_to_string(file) {
361 return Err(if let FromRedirect(true) = redirect {
362 LoadError::BrokenRedirect(file.to_path_buf(), err)
364 LoadError::IOError(err)
369 let maybe = maybe_redirect(&contents);
371 if let SkipRedirect = redirect {
372 return Err(LoadError::IsRedirect);
375 entry.insert(FileEntry { source: contents.clone(), ids: HashSet::new() });
380 match maybe_redirect.map(|url| file.parent().unwrap().join(url)) {
381 Some(redirect_file) => load_file(cache, root, &redirect_file, FromRedirect(true)),
382 None => Ok((pretty_file, contents)),
386 fn maybe_redirect(source: &str) -> Option<String> {
387 const REDIRECT: &str = "<p>Redirecting to <a href=";
389 let mut lines = source.lines();
390 let redirect_line = lines.nth(6)?;
392 redirect_line.find(REDIRECT).map(|i| {
393 let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
394 let pos_quote = rest.find('"').unwrap();
395 rest[..pos_quote].to_owned()
399 fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) {
401 for (i, mut line) in contents.lines().enumerate() {
402 while let Some(j) = line.find(attr) {
403 let rest = &line[j + attr.len()..];
404 // The base tag should always be the first link in the document so
405 // we can get away with using one pass.
406 let is_base = line[..j].ends_with("<base");
408 let pos_equals = match rest.find('=') {
412 if rest[..pos_equals].trim_start_matches(' ') != "" {
416 let rest = &rest[pos_equals + 1..];
418 let pos_quote = match rest.find(&['"', '\''][..]) {
422 let quote_delim = rest.as_bytes()[pos_quote] as char;
424 if rest[..pos_quote].trim_start_matches(' ') != "" {
427 let rest = &rest[pos_quote + 1..];
428 let url = match rest.find(quote_delim) {
429 Some(i) => &rest[..i],