src/bootstrap/doc.rs

   1 //! Documentation generation for rustbuilder.
   2 //!
   3 //! This module implements generation for all bits and pieces of documentation
   4 //! for the Rust project. This notably includes suites like the rust book, the
   5 //! nomicon, rust by example, standalone documentation, etc.
   6 //!
   7 //! Everything here is basically just a shim around calling either `rustbook` or
   8 //! `rustdoc`.
   9
  10 use std::ffi::OsStr;
  11 use std::fs;
  12 use std::io;
  13 use std::path::{Path, PathBuf};
  14
  15 use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step};
  16 use crate::cache::{Interned, INTERNER};
  17 use crate::compile;
  18 use crate::config::{Config, TargetSelection};
  19 use crate::tool::{self, prepare_tool_cargo, SourceType, Tool};
  20 use crate::util::{symlink_dir, t, up_to_date};
  21 use crate::Mode;
  22
  23 macro_rules! submodule_helper {
  24     ($path:expr, submodule) => {
  25         $path
  26     };
  27     ($path:expr, submodule = $submodule:literal) => {
  28         $submodule
  29     };
  30 }
  31
  32 macro_rules! book {
  33     ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => {
  34         $(
  35             #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  36         pub struct $name {
  37             target: TargetSelection,
  38         }
  39
  40         impl Step for $name {
  41             type Output = ();
  42             const DEFAULT: bool = true;
  43
  44             fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  45                 let builder = run.builder;
  46                 run.path($path).default_condition(builder.config.docs)
  47             }
  48
  49             fn make_run(run: RunConfig<'_>) {
  50                 run.builder.ensure($name {
  51                     target: run.target,
  52                 });
  53             }
  54
  55             fn run(self, builder: &Builder<'_>) {
  56                 $(
  57                     let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? ));
  58                     builder.update_submodule(&path);
  59                 )?
  60                 builder.ensure(RustbookSrc {
  61                     target: self.target,
  62                     name: INTERNER.intern_str($book_name),
  63                     src: INTERNER.intern_path(builder.src.join($path)),
  64                 })
  65             }
  66         }
  67         )+
  68     }
  69 }
  70
  71 // NOTE: When adding a book here, make sure to ALSO build the book by
  72 // adding a build step in `src/bootstrap/builder.rs`!
  73 // NOTE: Make sure to add the corresponding submodule when adding a new book.
  74 // FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules
  75 // and checking against it?).
  76 book!(
  77     CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo";
  78     ClippyBook, "src/tools/clippy/book", "clippy";
  79     EditionGuide, "src/doc/edition-guide", "edition-guide", submodule;
  80     EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule;
  81     Nomicon, "src/doc/nomicon", "nomicon", submodule;
  82     Reference, "src/doc/reference", "reference", submodule;
  83     RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule;
  84     RustdocBook, "src/doc/rustdoc", "rustdoc";
  85     StyleGuide, "src/doc/style-guide", "style-guide";
  86 );
  87
  88 // "library/std" -> ["library", "std"]
  89 //
  90 // Used for deciding whether a particular step is one requested by the user on
  91 // the `x.py doc` command line, which determines whether `--open` will open that
  92 // page.
  93 pub(crate) fn components_simplified(path: &PathBuf) -> Vec<&str> {
  94     path.iter().map(|component| component.to_str().unwrap_or("???")).collect()
  95 }
  96
  97 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  98 pub struct UnstableBook {
  99     target: TargetSelection,
 100 }
 101
 102 impl Step for UnstableBook {
 103     type Output = ();
 104     const DEFAULT: bool = true;
 105
 106     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 107         let builder = run.builder;
 108         run.path("src/doc/unstable-book").default_condition(builder.config.docs)
 109     }
 110
 111     fn make_run(run: RunConfig<'_>) {
 112         run.builder.ensure(UnstableBook { target: run.target });
 113     }
 114
 115     fn run(self, builder: &Builder<'_>) {
 116         builder.ensure(UnstableBookGen { target: self.target });
 117         builder.ensure(RustbookSrc {
 118             target: self.target,
 119             name: INTERNER.intern_str("unstable-book"),
 120             src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")),
 121         })
 122     }
 123 }
 124
 125 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 126 struct RustbookSrc {
 127     target: TargetSelection,
 128     name: Interned<String>,
 129     src: Interned<PathBuf>,
 130 }
 131
 132 impl Step for RustbookSrc {
 133     type Output = ();
 134
 135     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 136         run.never()
 137     }
 138
 139     /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.
 140     ///
 141     /// This will not actually generate any documentation if the documentation has
 142     /// already been generated.
 143     fn run(self, builder: &Builder<'_>) {
 144         let target = self.target;
 145         let name = self.name;
 146         let src = self.src;
 147         let out = builder.doc_out(target);
 148         t!(fs::create_dir_all(&out));
 149
 150         let out = out.join(name);
 151         let index = out.join("index.html");
 152         let rustbook = builder.tool_exe(Tool::Rustbook);
 153         let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
 154         if builder.config.dry_run() || up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
 155             return;
 156         }
 157         builder.info(&format!("Rustbook ({}) - {}", target, name));
 158         let _ = fs::remove_dir_all(&out);
 159
 160         builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out));
 161     }
 162 }
 163
 164 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 165 pub struct TheBook {
 166     compiler: Compiler,
 167     target: TargetSelection,
 168 }
 169
 170 impl Step for TheBook {
 171     type Output = ();
 172     const DEFAULT: bool = true;
 173
 174     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 175         let builder = run.builder;
 176         run.path("src/doc/book").default_condition(builder.config.docs)
 177     }
 178
 179     fn make_run(run: RunConfig<'_>) {
 180         run.builder.ensure(TheBook {
 181             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 182             target: run.target,
 183         });
 184     }
 185
 186     /// Builds the book and associated stuff.
 187     ///
 188     /// We need to build:
 189     ///
 190     /// * Book
 191     /// * Older edition redirects
 192     /// * Version info and CSS
 193     /// * Index page
 194     /// * Redirect pages
 195     fn run(self, builder: &Builder<'_>) {
 196         let relative_path = Path::new("src").join("doc").join("book");
 197         builder.update_submodule(&relative_path);
 198
 199         let compiler = self.compiler;
 200         let target = self.target;
 201
 202         // build book
 203         builder.ensure(RustbookSrc {
 204             target,
 205             name: INTERNER.intern_str("book"),
 206             src: INTERNER.intern_path(builder.src.join(&relative_path)),
 207         });
 208
 209         // building older edition redirects
 210         for edition in &["first-edition", "second-edition", "2018-edition"] {
 211             builder.ensure(RustbookSrc {
 212                 target,
 213                 name: INTERNER.intern_string(format!("book/{}", edition)),
 214                 src: INTERNER.intern_path(builder.src.join(&relative_path).join(edition)),
 215             });
 216         }
 217
 218         // build the version info page and CSS
 219         let shared_assets = builder.ensure(SharedAssets { target });
 220
 221         // build the redirect pages
 222         builder.info(&format!("Documenting book redirect pages ({})", target));
 223         for file in t!(fs::read_dir(builder.src.join(&relative_path).join("redirects"))) {
 224             let file = t!(file);
 225             let path = file.path();
 226             let path = path.to_str().unwrap();
 227
 228             invoke_rustdoc(builder, compiler, &shared_assets, target, path);
 229         }
 230
 231         let out = builder.doc_out(target);
 232         let index = out.join("book").join("index.html");
 233         builder.maybe_open_in_browser::<Self>(index);
 234     }
 235 }
 236
 237 fn invoke_rustdoc(
 238     builder: &Builder<'_>,
 239     compiler: Compiler,
 240     shared_assets: &SharedAssetsPaths,
 241     target: TargetSelection,
 242     markdown: &str,
 243 ) {
 244     let out = builder.doc_out(target);
 245
 246     let path = builder.src.join("src/doc").join(markdown);
 247
 248     let header = builder.src.join("src/doc/redirect.inc");
 249     let footer = builder.src.join("src/doc/footer.inc");
 250
 251     let mut cmd = builder.rustdoc_cmd(compiler);
 252
 253     let out = out.join("book");
 254
 255     cmd.arg("--html-after-content")
 256         .arg(&footer)
 257         .arg("--html-before-content")
 258         .arg(&shared_assets.version_info)
 259         .arg("--html-in-header")
 260         .arg(&header)
 261         .arg("--markdown-no-toc")
 262         .arg("--markdown-playground-url")
 263         .arg("https://play.rust-lang.org/")
 264         .arg("-o")
 265         .arg(&out)
 266         .arg(&path)
 267         .arg("--markdown-css")
 268         .arg("../rust.css");
 269
 270     if !builder.config.docs_minification {
 271         cmd.arg("-Z").arg("unstable-options").arg("--disable-minification");
 272     }
 273
 274     builder.run(&mut cmd);
 275 }
 276
 277 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 278 pub struct Standalone {
 279     compiler: Compiler,
 280     target: TargetSelection,
 281 }
 282
 283 impl Step for Standalone {
 284     type Output = ();
 285     const DEFAULT: bool = true;
 286
 287     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 288         let builder = run.builder;
 289         run.path("src/doc").alias("standalone").default_condition(builder.config.docs)
 290     }
 291
 292     fn make_run(run: RunConfig<'_>) {
 293         run.builder.ensure(Standalone {
 294             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 295             target: run.target,
 296         });
 297     }
 298
 299     /// Generates all standalone documentation as compiled by the rustdoc in `stage`
 300     /// for the `target` into `out`.
 301     ///
 302     /// This will list all of `src/doc` looking for markdown files and appropriately
 303     /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and
 304     /// `STAMP` along with providing the various header/footer HTML we've customized.
 305     ///
 306     /// In the end, this is just a glorified wrapper around rustdoc!
 307     fn run(self, builder: &Builder<'_>) {
 308         let target = self.target;
 309         let compiler = self.compiler;
 310         builder.info(&format!("Documenting standalone ({})", target));
 311         let out = builder.doc_out(target);
 312         t!(fs::create_dir_all(&out));
 313
 314         let version_info = builder.ensure(SharedAssets { target: self.target }).version_info;
 315
 316         let favicon = builder.src.join("src/doc/favicon.inc");
 317         let footer = builder.src.join("src/doc/footer.inc");
 318         let full_toc = builder.src.join("src/doc/full-toc.inc");
 319
 320         for file in t!(fs::read_dir(builder.src.join("src/doc"))) {
 321             let file = t!(file);
 322             let path = file.path();
 323             let filename = path.file_name().unwrap().to_str().unwrap();
 324             if !filename.ends_with(".md") || filename == "README.md" {
 325                 continue;
 326             }
 327
 328             let html = out.join(filename).with_extension("html");
 329             let rustdoc = builder.rustdoc(compiler);
 330             if up_to_date(&path, &html)
 331                 && up_to_date(&footer, &html)
 332                 && up_to_date(&favicon, &html)
 333                 && up_to_date(&full_toc, &html)
 334                 && (builder.config.dry_run() || up_to_date(&version_info, &html))
 335                 && (builder.config.dry_run() || up_to_date(&rustdoc, &html))
 336             {
 337                 continue;
 338             }
 339
 340             let mut cmd = builder.rustdoc_cmd(compiler);
 341             // Needed for --index-page flag
 342             cmd.arg("-Z").arg("unstable-options");
 343
 344             cmd.arg("--html-after-content")
 345                 .arg(&footer)
 346                 .arg("--html-before-content")
 347                 .arg(&version_info)
 348                 .arg("--html-in-header")
 349                 .arg(&favicon)
 350                 .arg("--markdown-no-toc")
 351                 .arg("--index-page")
 352                 .arg(&builder.src.join("src/doc/index.md"))
 353                 .arg("--markdown-playground-url")
 354                 .arg("https://play.rust-lang.org/")
 355                 .arg("-o")
 356                 .arg(&out)
 357                 .arg(&path);
 358
 359             if !builder.config.docs_minification {
 360                 cmd.arg("--disable-minification");
 361             }
 362
 363             if filename == "not_found.md" {
 364                 cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css");
 365             } else {
 366                 cmd.arg("--markdown-css").arg("rust.css");
 367             }
 368             builder.run(&mut cmd);
 369         }
 370
 371         // We open doc/index.html as the default if invoked as `x.py doc --open`
 372         // with no particular explicit doc requested (e.g. library/core).
 373         if builder.paths.is_empty() || builder.was_invoked_explicitly::<Self>(Kind::Doc) {
 374             let index = out.join("index.html");
 375             builder.open_in_browser(&index);
 376         }
 377     }
 378 }
 379
 380 #[derive(Debug, Clone)]
 381 pub struct SharedAssetsPaths {
 382     pub version_info: PathBuf,
 383 }
 384
 385 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 386 pub struct SharedAssets {
 387     target: TargetSelection,
 388 }
 389
 390 impl Step for SharedAssets {
 391     type Output = SharedAssetsPaths;
 392     const DEFAULT: bool = false;
 393
 394     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 395         // Other tasks depend on this, no need to execute it on its own
 396         run.never()
 397     }
 398
 399     // Generate shared resources used by other pieces of documentation.
 400     fn run(self, builder: &Builder<'_>) -> Self::Output {
 401         let out = builder.doc_out(self.target);
 402
 403         let version_input = builder.src.join("src").join("doc").join("version_info.html.template");
 404         let version_info = out.join("version_info.html");
 405         if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) {
 406             let info = t!(fs::read_to_string(&version_input))
 407                 .replace("VERSION", &builder.rust_release())
 408                 .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or(""))
 409                 .replace("STAMP", builder.rust_info().sha().unwrap_or(""));
 410             t!(fs::write(&version_info, &info));
 411         }
 412
 413         builder.copy(&builder.src.join("src").join("doc").join("rust.css"), &out.join("rust.css"));
 414
 415         SharedAssetsPaths { version_info }
 416     }
 417 }
 418
 419 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 420 pub struct Std {
 421     pub stage: u32,
 422     pub target: TargetSelection,
 423     pub format: DocumentationFormat,
 424 }
 425
 426 impl Step for Std {
 427     type Output = ();
 428     const DEFAULT: bool = true;
 429
 430     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 431         let builder = run.builder;
 432         run.all_krates("test").path("library").default_condition(builder.config.docs)
 433     }
 434
 435     fn make_run(run: RunConfig<'_>) {
 436         run.builder.ensure(Std {
 437             stage: run.builder.top_stage,
 438             target: run.target,
 439             format: if run.builder.config.cmd.json() {
 440                 DocumentationFormat::JSON
 441             } else {
 442                 DocumentationFormat::HTML
 443             },
 444         });
 445     }
 446
 447     /// Compile all standard library documentation.
 448     ///
 449     /// This will generate all documentation for the standard library and its
 450     /// dependencies. This is largely just a wrapper around `cargo doc`.
 451     fn run(self, builder: &Builder<'_>) {
 452         let stage = self.stage;
 453         let target = self.target;
 454         let out = match self.format {
 455             DocumentationFormat::HTML => builder.doc_out(target),
 456             DocumentationFormat::JSON => builder.json_doc_out(target),
 457         };
 458
 459         t!(fs::create_dir_all(&out));
 460
 461         if self.format == DocumentationFormat::HTML {
 462             builder.ensure(SharedAssets { target: self.target });
 463         }
 464
 465         let index_page = builder.src.join("src/doc/index.md").into_os_string();
 466         let mut extra_args = match self.format {
 467             DocumentationFormat::HTML => vec![
 468                 OsStr::new("--markdown-css"),
 469                 OsStr::new("rust.css"),
 470                 OsStr::new("--markdown-no-toc"),
 471                 OsStr::new("--index-page"),
 472                 &index_page,
 473             ],
 474             DocumentationFormat::JSON => vec![OsStr::new("--output-format"), OsStr::new("json")],
 475         };
 476
 477         if !builder.config.docs_minification {
 478             extra_args.push(OsStr::new("--disable-minification"));
 479         }
 480
 481         let requested_crates = builder
 482             .paths
 483             .iter()
 484             .map(components_simplified)
 485             .filter_map(|path| {
 486                 if path.len() >= 2 && path.get(0) == Some(&"library") {
 487                     // single crate
 488                     Some(path[1].to_owned())
 489                 } else if !path.is_empty() {
 490                     // ??
 491                     Some(path[0].to_owned())
 492                 } else {
 493                     // all library crates
 494                     None
 495                 }
 496             })
 497             .collect::<Vec<_>>();
 498
 499         doc_std(builder, self.format, stage, target, &out, &extra_args, &requested_crates);
 500
 501         // Don't open if the format is json
 502         if let DocumentationFormat::JSON = self.format {
 503             return;
 504         }
 505
 506         // Look for library/std, library/core etc in the `x.py doc` arguments and
 507         // open the corresponding rendered docs.
 508         for requested_crate in requested_crates {
 509             if requested_crate == "library" {
 510                 // For `x.py doc library --open`, open `std` by default.
 511                 let index = out.join("std").join("index.html");
 512                 builder.open_in_browser(index);
 513             } else if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) {
 514                 let index = out.join(requested_crate).join("index.html");
 515                 builder.open_in_browser(index);
 516             }
 517         }
 518     }
 519 }
 520
 521 /// Name of the crates that are visible to consumers of the standard library.
 522 /// Documentation for internal crates is handled by the rustc step, so internal crates will show
 523 /// up there.
 524 ///
 525 /// Order here is important!
 526 /// Crates need to be processed starting from the leaves, otherwise rustdoc will not
 527 /// create correct links between crates because rustdoc depends on the
 528 /// existence of the output directories to know if it should be a local
 529 /// or remote link.
 530 const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"];
 531
 532 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 533 pub enum DocumentationFormat {
 534     HTML,
 535     JSON,
 536 }
 537
 538 impl DocumentationFormat {
 539     fn as_str(&self) -> &str {
 540         match self {
 541             DocumentationFormat::HTML => "HTML",
 542             DocumentationFormat::JSON => "JSON",
 543         }
 544     }
 545 }
 546
 547 /// Build the documentation for public standard library crates.
 548 ///
 549 /// `requested_crates` can be used to build only a subset of the crates. If empty, all crates will
 550 /// be built.
 551 fn doc_std(
 552     builder: &Builder<'_>,
 553     format: DocumentationFormat,
 554     stage: u32,
 555     target: TargetSelection,
 556     out: &Path,
 557     extra_args: &[&OsStr],
 558     requested_crates: &[String],
 559 ) {
 560     builder.info(&format!(
 561         "Documenting stage{} std ({}) in {} format",
 562         stage,
 563         target,
 564         format.as_str()
 565     ));
 566     if builder.no_std(target) == Some(true) {
 567         panic!(
 568             "building std documentation for no_std target {target} is not supported\n\
 569              Set `docs = false` in the config to disable documentation."
 570         );
 571     }
 572     let compiler = builder.compiler(stage, builder.config.build);
 573
 574     let target_doc_dir_name = if format == DocumentationFormat::JSON { "json-doc" } else { "doc" };
 575     let target_dir =
 576         builder.stage_out(compiler, Mode::Std).join(target.triple).join(target_doc_dir_name);
 577
 578     // This is directory where the compiler will place the output of the command.
 579     // We will then copy the files from this directory into the final `out` directory, the specified
 580     // as a function parameter.
 581     let out_dir = target_dir.join(target.triple).join("doc");
 582
 583     let run_cargo_rustdoc_for = |package: &str| {
 584         let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc");
 585         compile::std_cargo(builder, target, compiler.stage, &mut cargo);
 586         cargo
 587             .arg("--target-dir")
 588             .arg(&*target_dir.to_string_lossy())
 589             .arg("-p")
 590             .arg(package)
 591             .arg("-Zskip-rustdoc-fingerprint")
 592             .arg("--")
 593             .arg("-Z")
 594             .arg("unstable-options")
 595             .arg("--resource-suffix")
 596             .arg(&builder.version)
 597             .args(extra_args);
 598         builder.run(&mut cargo.into());
 599     };
 600
 601     for krate in STD_PUBLIC_CRATES {
 602         run_cargo_rustdoc_for(krate);
 603         if requested_crates.iter().any(|p| p == krate) {
 604             // No need to document more of the libraries if we have the one we want.
 605             break;
 606         }
 607     }
 608
 609     builder.cp_r(&out_dir, &out);
 610 }
 611
 612 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 613 pub struct Rustc {
 614     pub stage: u32,
 615     pub target: TargetSelection,
 616 }
 617
 618 impl Step for Rustc {
 619     type Output = ();
 620     const DEFAULT: bool = true;
 621     const ONLY_HOSTS: bool = true;
 622
 623     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 624         let builder = run.builder;
 625         run.crate_or_deps("rustc-main")
 626             .path("compiler")
 627             .default_condition(builder.config.compiler_docs)
 628     }
 629
 630     fn make_run(run: RunConfig<'_>) {
 631         run.builder.ensure(Rustc { stage: run.builder.top_stage, target: run.target });
 632     }
 633
 634     /// Generates compiler documentation.
 635     ///
 636     /// This will generate all documentation for compiler and dependencies.
 637     /// Compiler documentation is distributed separately, so we make sure
 638     /// we do not merge it with the other documentation from std, test and
 639     /// proc_macros. This is largely just a wrapper around `cargo doc`.
 640     fn run(self, builder: &Builder<'_>) {
 641         let stage = self.stage;
 642         let target = self.target;
 643
 644         let paths = builder
 645             .paths
 646             .iter()
 647             .filter(|path| {
 648                 let components = components_simplified(path);
 649                 components.len() >= 2 && components[0] == "compiler"
 650             })
 651             .collect::<Vec<_>>();
 652
 653         // This is the intended out directory for compiler documentation.
 654         let out = builder.compiler_doc_out(target);
 655         t!(fs::create_dir_all(&out));
 656
 657         // Build the standard library, so that proc-macros can use it.
 658         // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.)
 659         let compiler = builder.compiler(stage, builder.config.build);
 660         builder.ensure(compile::Std::new(compiler, builder.config.build));
 661
 662         builder.info(&format!("Documenting stage{} compiler ({})", stage, target));
 663
 664         // This uses a shared directory so that librustdoc documentation gets
 665         // correctly built and merged with the rustc documentation. This is
 666         // needed because rustdoc is built in a different directory from
 667         // rustc. rustdoc needs to be able to see everything, for example when
 668         // merging the search index, or generating local (relative) links.
 669         let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc");
 670         t!(symlink_dir_force(&builder.config, &out, &out_dir));
 671         // Cargo puts proc macros in `target/doc` even if you pass `--target`
 672         // explicitly (https://github.com/rust-lang/cargo/issues/7677).
 673         let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc");
 674         t!(symlink_dir_force(&builder.config, &out, &proc_macro_out_dir));
 675
 676         // Build cargo command.
 677         let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc");
 678         cargo.rustdocflag("--document-private-items");
 679         // Since we always pass --document-private-items, there's no need to warn about linking to private items.
 680         cargo.rustdocflag("-Arustdoc::private-intra-doc-links");
 681         cargo.rustdocflag("--enable-index-page");
 682         cargo.rustdocflag("-Zunstable-options");
 683         cargo.rustdocflag("-Znormalize-docs");
 684         cargo.rustdocflag("--show-type-layout");
 685         cargo.rustdocflag("--generate-link-to-definition");
 686         compile::rustc_cargo(builder, &mut cargo, target);
 687         cargo.arg("-Zunstable-options");
 688         cargo.arg("-Zskip-rustdoc-fingerprint");
 689
 690         // Only include compiler crates, no dependencies of those, such as `libc`.
 691         // Do link to dependencies on `docs.rs` however using `rustdoc-map`.
 692         cargo.arg("--no-deps");
 693         cargo.arg("-Zrustdoc-map");
 694
 695         // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies,
 696         // once this is no longer an issue the special case for `ena` can be removed.
 697         cargo.rustdocflag("--extern-html-root-url");
 698         cargo.rustdocflag("ena=https://docs.rs/ena/latest/");
 699
 700         let root_crates = if paths.is_empty() {
 701             vec![
 702                 INTERNER.intern_str("rustc_driver"),
 703                 INTERNER.intern_str("rustc_codegen_llvm"),
 704                 INTERNER.intern_str("rustc_codegen_ssa"),
 705             ]
 706         } else {
 707             paths.into_iter().map(|p| builder.crate_paths[p]).collect()
 708         };
 709         // Find dependencies for top level crates.
 710         let compiler_crates = root_crates.iter().flat_map(|krate| {
 711             builder.in_tree_crates(krate, Some(target)).into_iter().map(|krate| krate.name)
 712         });
 713
 714         let mut to_open = None;
 715         for krate in compiler_crates {
 716             // Create all crate output directories first to make sure rustdoc uses
 717             // relative links.
 718             // FIXME: Cargo should probably do this itself.
 719             t!(fs::create_dir_all(out_dir.join(krate)));
 720             cargo.arg("-p").arg(krate);
 721             if to_open.is_none() {
 722                 to_open = Some(krate);
 723             }
 724         }
 725
 726         builder.run(&mut cargo.into());
 727         // Let's open the first crate documentation page:
 728         if let Some(krate) = to_open {
 729             let index = out.join(krate).join("index.html");
 730             builder.open_in_browser(index);
 731         }
 732     }
 733 }
 734
 735 macro_rules! tool_doc {
 736     ($tool: ident, $should_run: literal, $path: literal, $(rustc_tool = $rustc_tool:literal, )? $(in_tree = $in_tree:literal, )? [$($krate: literal),+ $(,)?] $(,)?) => {
 737         #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 738         pub struct $tool {
 739             target: TargetSelection,
 740         }
 741
 742         impl Step for $tool {
 743             type Output = ();
 744             const DEFAULT: bool = true;
 745             const ONLY_HOSTS: bool = true;
 746
 747             fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 748                 let builder = run.builder;
 749                 run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs)
 750             }
 751
 752             fn make_run(run: RunConfig<'_>) {
 753                 run.builder.ensure($tool { target: run.target });
 754             }
 755
 756             /// Generates compiler documentation.
 757             ///
 758             /// This will generate all documentation for compiler and dependencies.
 759             /// Compiler documentation is distributed separately, so we make sure
 760             /// we do not merge it with the other documentation from std, test and
 761             /// proc_macros. This is largely just a wrapper around `cargo doc`.
 762             fn run(self, builder: &Builder<'_>) {
 763                 let stage = builder.top_stage;
 764                 let target = self.target;
 765
 766                 // This is the intended out directory for compiler documentation.
 767                 let out = builder.compiler_doc_out(target);
 768                 t!(fs::create_dir_all(&out));
 769
 770                 let compiler = builder.compiler(stage, builder.config.build);
 771                 builder.ensure(compile::Std::new(compiler, target));
 772
 773                 if true $(&& $rustc_tool)? {
 774                     // Build rustc docs so that we generate relative links.
 775                     builder.ensure(Rustc { stage, target });
 776
 777                     // Rustdoc needs the rustc sysroot available to build.
 778                     // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed
 779                     // with strange errors, but only on a full bors test ...
 780                     builder.ensure(compile::Rustc::new(compiler, target));
 781                 }
 782
 783                 let source_type = if true $(&& $in_tree)? {
 784                     SourceType::InTree
 785                 } else {
 786                     SourceType::Submodule
 787                 };
 788
 789                 builder.info(
 790                     &format!(
 791                         "Documenting stage{} {} ({})",
 792                         stage,
 793                         stringify!($tool).to_lowercase(),
 794                         target,
 795                     ),
 796                 );
 797
 798                 // Symlink compiler docs to the output directory of rustdoc documentation.
 799                 let out_dirs = [
 800                     builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"),
 801                     // Cargo uses a different directory for proc macros.
 802                     builder.stage_out(compiler, Mode::ToolRustc).join("doc"),
 803                 ];
 804                 for out_dir in out_dirs {
 805                     t!(fs::create_dir_all(&out_dir));
 806                     t!(symlink_dir_force(&builder.config, &out, &out_dir));
 807                 }
 808
 809                 // Build cargo command.
 810                 let mut cargo = prepare_tool_cargo(
 811                     builder,
 812                     compiler,
 813                     Mode::ToolRustc,
 814                     target,
 815                     "doc",
 816                     $path,
 817                     source_type,
 818                     &[],
 819                 );
 820
 821                 cargo.arg("-Zskip-rustdoc-fingerprint");
 822                 // Only include compiler crates, no dependencies of those, such as `libc`.
 823                 cargo.arg("--no-deps");
 824                 $(
 825                     cargo.arg("-p").arg($krate);
 826                 )+
 827
 828                 cargo.rustdocflag("--document-private-items");
 829                 cargo.rustdocflag("--enable-index-page");
 830                 cargo.rustdocflag("--show-type-layout");
 831                 cargo.rustdocflag("--generate-link-to-definition");
 832                 cargo.rustdocflag("-Zunstable-options");
 833                 builder.run(&mut cargo.into());
 834             }
 835         }
 836     }
 837 }
 838
 839 tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", ["rustdoc", "rustdoc-json-types"],);
 840 tool_doc!(
 841     Rustfmt,
 842     "rustfmt-nightly",
 843     "src/tools/rustfmt",
 844     ["rustfmt-nightly", "rustfmt-config_proc_macro"],
 845 );
 846 tool_doc!(Clippy, "clippy", "src/tools/clippy", ["clippy_utils"]);
 847 tool_doc!(Miri, "miri", "src/tools/miri", ["miri"]);
 848 tool_doc!(
 849     Cargo,
 850     "cargo",
 851     "src/tools/cargo",
 852     rustc_tool = false,
 853     in_tree = false,
 854     [
 855         "cargo",
 856         "cargo-platform",
 857         "cargo-util",
 858         "crates-io",
 859         "cargo-test-macro",
 860         "cargo-test-support",
 861         "cargo-credential",
 862         "cargo-credential-1password",
 863         "mdman",
 864         // FIXME: this trips a license check in tidy.
 865         // "resolver-tests",
 866         // FIXME: we should probably document these, but they're different per-platform so we can't use `tool_doc`.
 867         // "cargo-credential-gnome-secret",
 868         // "cargo-credential-macos-keychain",
 869         // "cargo-credential-wincred",
 870     ]
 871 );
 872
 873 #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]
 874 pub struct ErrorIndex {
 875     pub target: TargetSelection,
 876 }
 877
 878 impl Step for ErrorIndex {
 879     type Output = ();
 880     const DEFAULT: bool = true;
 881     const ONLY_HOSTS: bool = true;
 882
 883     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 884         let builder = run.builder;
 885         run.path("src/tools/error_index_generator").default_condition(builder.config.docs)
 886     }
 887
 888     fn make_run(run: RunConfig<'_>) {
 889         let target = run.target;
 890         run.builder.ensure(ErrorIndex { target });
 891     }
 892
 893     /// Generates the HTML rendered error-index by running the
 894     /// `error_index_generator` tool.
 895     fn run(self, builder: &Builder<'_>) {
 896         builder.info(&format!("Documenting error index ({})", self.target));
 897         let out = builder.doc_out(self.target);
 898         t!(fs::create_dir_all(&out));
 899         let mut index = tool::ErrorIndex::command(builder);
 900         index.arg("html");
 901         index.arg(out);
 902         index.arg(&builder.version);
 903
 904         builder.run(&mut index);
 905     }
 906 }
 907
 908 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 909 pub struct UnstableBookGen {
 910     target: TargetSelection,
 911 }
 912
 913 impl Step for UnstableBookGen {
 914     type Output = ();
 915     const DEFAULT: bool = true;
 916     const ONLY_HOSTS: bool = true;
 917
 918     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 919         let builder = run.builder;
 920         run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs)
 921     }
 922
 923     fn make_run(run: RunConfig<'_>) {
 924         run.builder.ensure(UnstableBookGen { target: run.target });
 925     }
 926
 927     fn run(self, builder: &Builder<'_>) {
 928         let target = self.target;
 929
 930         builder.info(&format!("Generating unstable book md files ({})", target));
 931         let out = builder.md_doc_out(target).join("unstable-book");
 932         builder.create_dir(&out);
 933         builder.remove_dir(&out);
 934         let mut cmd = builder.tool_cmd(Tool::UnstableBookGen);
 935         cmd.arg(builder.src.join("library"));
 936         cmd.arg(builder.src.join("compiler"));
 937         cmd.arg(builder.src.join("src"));
 938         cmd.arg(out);
 939
 940         builder.run(&mut cmd);
 941     }
 942 }
 943
 944 fn symlink_dir_force(config: &Config, src: &Path, dst: &Path) -> io::Result<()> {
 945     if config.dry_run() {
 946         return Ok(());
 947     }
 948     if let Ok(m) = fs::symlink_metadata(dst) {
 949         if m.file_type().is_dir() {
 950             fs::remove_dir_all(dst)?;
 951         } else {
 952             // handle directory junctions on windows by falling back to
 953             // `remove_dir`.
 954             fs::remove_file(dst).or_else(|_| fs::remove_dir(dst))?;
 955         }
 956     }
 957
 958     symlink_dir(config, src, dst)
 959 }
 960
 961 #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]
 962 pub struct RustcBook {
 963     pub compiler: Compiler,
 964     pub target: TargetSelection,
 965     pub validate: bool,
 966 }
 967
 968 impl Step for RustcBook {
 969     type Output = ();
 970     const DEFAULT: bool = true;
 971     const ONLY_HOSTS: bool = true;
 972
 973     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 974         let builder = run.builder;
 975         run.path("src/doc/rustc").default_condition(builder.config.docs)
 976     }
 977
 978     fn make_run(run: RunConfig<'_>) {
 979         run.builder.ensure(RustcBook {
 980             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 981             target: run.target,
 982             validate: false,
 983         });
 984     }
 985
 986     /// Builds the rustc book.
 987     ///
 988     /// The lints are auto-generated by a tool, and then merged into the book
 989     /// in the "md-doc" directory in the build output directory. Then
 990     /// "rustbook" is used to convert it to HTML.
 991     fn run(self, builder: &Builder<'_>) {
 992         let out_base = builder.md_doc_out(self.target).join("rustc");
 993         t!(fs::create_dir_all(&out_base));
 994         let out_listing = out_base.join("src/lints");
 995         builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base);
 996         builder.info(&format!("Generating lint docs ({})", self.target));
 997
 998         let rustc = builder.rustc(self.compiler);
 999         // The tool runs `rustc` for extracting output examples, so it needs a
1000         // functional sysroot.
1001         builder.ensure(compile::Std::new(self.compiler, self.target));
1002         let mut cmd = builder.tool_cmd(Tool::LintDocs);
1003         cmd.arg("--src");
1004         cmd.arg(builder.src.join("compiler"));
1005         cmd.arg("--out");
1006         cmd.arg(&out_listing);
1007         cmd.arg("--rustc");
1008         cmd.arg(&rustc);
1009         cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg());
1010         if builder.is_verbose() {
1011             cmd.arg("--verbose");
1012         }
1013         if self.validate {
1014             cmd.arg("--validate");
1015         }
1016         if !builder.unstable_features() {
1017             // We need to validate nightly features, even on the stable channel.
1018             cmd.env("RUSTC_BOOTSTRAP", "1");
1019         }
1020         // If the lib directories are in an unusual location (changed in
1021         // config.toml), then this needs to explicitly update the dylib search
1022         // path.
1023         builder.add_rustc_lib_path(self.compiler, &mut cmd);
1024         builder.run(&mut cmd);
1025         // Run rustbook/mdbook to generate the HTML pages.
1026         builder.ensure(RustbookSrc {
1027             target: self.target,
1028             name: INTERNER.intern_str("rustc"),
1029             src: INTERNER.intern_path(out_base),
1030         });
1031
1032         let out = builder.doc_out(self.target);
1033         let index = out.join("rustc").join("index.html");
1034         builder.maybe_open_in_browser::<Self>(index);
1035     }
1036 }