src/bootstrap/doc.rs

   1 //! Documentation generation for rustbuilder.
   2 //!
   3 //! This module implements generation for all bits and pieces of documentation
   4 //! for the Rust project. This notably includes suites like the rust book, the
   5 //! nomicon, rust by example, standalone documentation, etc.
   6 //!
   7 //! Everything here is basically just a shim around calling either `rustbook` or
   8 //! `rustdoc`.
   9
  10 use std::ffi::OsStr;
  11 use std::fs;
  12 use std::io;
  13 use std::path::{Path, PathBuf};
  14
  15 use crate::builder::crate_description;
  16 use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step};
  17 use crate::cache::{Interned, INTERNER};
  18 use crate::compile;
  19 use crate::config::{Config, TargetSelection};
  20 use crate::tool::{self, prepare_tool_cargo, SourceType, Tool};
  21 use crate::util::{symlink_dir, t, up_to_date};
  22 use crate::Mode;
  23
  24 macro_rules! submodule_helper {
  25     ($path:expr, submodule) => {
  26         $path
  27     };
  28     ($path:expr, submodule = $submodule:literal) => {
  29         $submodule
  30     };
  31 }
  32
  33 macro_rules! book {
  34     ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => {
  35         $(
  36             #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  37         pub struct $name {
  38             target: TargetSelection,
  39         }
  40
  41         impl Step for $name {
  42             type Output = ();
  43             const DEFAULT: bool = true;
  44
  45             fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  46                 let builder = run.builder;
  47                 run.path($path).default_condition(builder.config.docs)
  48             }
  49
  50             fn make_run(run: RunConfig<'_>) {
  51                 run.builder.ensure($name {
  52                     target: run.target,
  53                 });
  54             }
  55
  56             fn run(self, builder: &Builder<'_>) {
  57                 $(
  58                     let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? ));
  59                     builder.update_submodule(&path);
  60                 )?
  61                 builder.ensure(RustbookSrc {
  62                     target: self.target,
  63                     name: INTERNER.intern_str($book_name),
  64                     src: INTERNER.intern_path(builder.src.join($path)),
  65                 })
  66             }
  67         }
  68         )+
  69     }
  70 }
  71
  72 // NOTE: When adding a book here, make sure to ALSO build the book by
  73 // adding a build step in `src/bootstrap/builder.rs`!
  74 // NOTE: Make sure to add the corresponding submodule when adding a new book.
  75 // FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules
  76 // and checking against it?).
  77 book!(
  78     CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo";
  79     ClippyBook, "src/tools/clippy/book", "clippy";
  80     EditionGuide, "src/doc/edition-guide", "edition-guide", submodule;
  81     EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule;
  82     Nomicon, "src/doc/nomicon", "nomicon", submodule;
  83     Reference, "src/doc/reference", "reference", submodule;
  84     RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule;
  85     RustdocBook, "src/doc/rustdoc", "rustdoc";
  86     StyleGuide, "src/doc/style-guide", "style-guide";
  87 );
  88
  89 // "library/std" -> ["library", "std"]
  90 //
  91 // Used for deciding whether a particular step is one requested by the user on
  92 // the `x.py doc` command line, which determines whether `--open` will open that
  93 // page.
  94 pub(crate) fn components_simplified(path: &PathBuf) -> Vec<&str> {
  95     path.iter().map(|component| component.to_str().unwrap_or("???")).collect()
  96 }
  97
  98 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  99 pub struct UnstableBook {
 100     target: TargetSelection,
 101 }
 102
 103 impl Step for UnstableBook {
 104     type Output = ();
 105     const DEFAULT: bool = true;
 106
 107     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 108         let builder = run.builder;
 109         run.path("src/doc/unstable-book").default_condition(builder.config.docs)
 110     }
 111
 112     fn make_run(run: RunConfig<'_>) {
 113         run.builder.ensure(UnstableBook { target: run.target });
 114     }
 115
 116     fn run(self, builder: &Builder<'_>) {
 117         builder.ensure(UnstableBookGen { target: self.target });
 118         builder.ensure(RustbookSrc {
 119             target: self.target,
 120             name: INTERNER.intern_str("unstable-book"),
 121             src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")),
 122         })
 123     }
 124 }
 125
 126 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 127 struct RustbookSrc {
 128     target: TargetSelection,
 129     name: Interned<String>,
 130     src: Interned<PathBuf>,
 131 }
 132
 133 impl Step for RustbookSrc {
 134     type Output = ();
 135
 136     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 137         run.never()
 138     }
 139
 140     /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.
 141     ///
 142     /// This will not actually generate any documentation if the documentation has
 143     /// already been generated.
 144     fn run(self, builder: &Builder<'_>) {
 145         let target = self.target;
 146         let name = self.name;
 147         let src = self.src;
 148         let out = builder.doc_out(target);
 149         t!(fs::create_dir_all(&out));
 150
 151         let out = out.join(name);
 152         let index = out.join("index.html");
 153         let rustbook = builder.tool_exe(Tool::Rustbook);
 154         let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
 155         if builder.config.dry_run() || up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
 156             return;
 157         }
 158         builder.info(&format!("Rustbook ({}) - {}", target, name));
 159         let _ = fs::remove_dir_all(&out);
 160
 161         builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out));
 162     }
 163 }
 164
 165 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 166 pub struct TheBook {
 167     compiler: Compiler,
 168     target: TargetSelection,
 169 }
 170
 171 impl Step for TheBook {
 172     type Output = ();
 173     const DEFAULT: bool = true;
 174
 175     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 176         let builder = run.builder;
 177         run.path("src/doc/book").default_condition(builder.config.docs)
 178     }
 179
 180     fn make_run(run: RunConfig<'_>) {
 181         run.builder.ensure(TheBook {
 182             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 183             target: run.target,
 184         });
 185     }
 186
 187     /// Builds the book and associated stuff.
 188     ///
 189     /// We need to build:
 190     ///
 191     /// * Book
 192     /// * Older edition redirects
 193     /// * Version info and CSS
 194     /// * Index page
 195     /// * Redirect pages
 196     fn run(self, builder: &Builder<'_>) {
 197         let relative_path = Path::new("src").join("doc").join("book");
 198         builder.update_submodule(&relative_path);
 199
 200         let compiler = self.compiler;
 201         let target = self.target;
 202
 203         // build book
 204         builder.ensure(RustbookSrc {
 205             target,
 206             name: INTERNER.intern_str("book"),
 207             src: INTERNER.intern_path(builder.src.join(&relative_path)),
 208         });
 209
 210         // building older edition redirects
 211         for edition in &["first-edition", "second-edition", "2018-edition"] {
 212             builder.ensure(RustbookSrc {
 213                 target,
 214                 name: INTERNER.intern_string(format!("book/{}", edition)),
 215                 src: INTERNER.intern_path(builder.src.join(&relative_path).join(edition)),
 216             });
 217         }
 218
 219         // build the version info page and CSS
 220         let shared_assets = builder.ensure(SharedAssets { target });
 221
 222         // build the redirect pages
 223         builder.info(&format!("Documenting book redirect pages ({})", target));
 224         for file in t!(fs::read_dir(builder.src.join(&relative_path).join("redirects"))) {
 225             let file = t!(file);
 226             let path = file.path();
 227             let path = path.to_str().unwrap();
 228
 229             invoke_rustdoc(builder, compiler, &shared_assets, target, path);
 230         }
 231
 232         let out = builder.doc_out(target);
 233         let index = out.join("book").join("index.html");
 234         builder.maybe_open_in_browser::<Self>(index);
 235     }
 236 }
 237
 238 fn invoke_rustdoc(
 239     builder: &Builder<'_>,
 240     compiler: Compiler,
 241     shared_assets: &SharedAssetsPaths,
 242     target: TargetSelection,
 243     markdown: &str,
 244 ) {
 245     let out = builder.doc_out(target);
 246
 247     let path = builder.src.join("src/doc").join(markdown);
 248
 249     let header = builder.src.join("src/doc/redirect.inc");
 250     let footer = builder.src.join("src/doc/footer.inc");
 251
 252     let mut cmd = builder.rustdoc_cmd(compiler);
 253
 254     let out = out.join("book");
 255
 256     cmd.arg("--html-after-content")
 257         .arg(&footer)
 258         .arg("--html-before-content")
 259         .arg(&shared_assets.version_info)
 260         .arg("--html-in-header")
 261         .arg(&header)
 262         .arg("--markdown-no-toc")
 263         .arg("--markdown-playground-url")
 264         .arg("https://play.rust-lang.org/")
 265         .arg("-o")
 266         .arg(&out)
 267         .arg(&path)
 268         .arg("--markdown-css")
 269         .arg("../rust.css");
 270
 271     if !builder.config.docs_minification {
 272         cmd.arg("-Z").arg("unstable-options").arg("--disable-minification");
 273     }
 274
 275     builder.run(&mut cmd);
 276 }
 277
 278 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 279 pub struct Standalone {
 280     compiler: Compiler,
 281     target: TargetSelection,
 282 }
 283
 284 impl Step for Standalone {
 285     type Output = ();
 286     const DEFAULT: bool = true;
 287
 288     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 289         let builder = run.builder;
 290         run.path("src/doc").alias("standalone").default_condition(builder.config.docs)
 291     }
 292
 293     fn make_run(run: RunConfig<'_>) {
 294         run.builder.ensure(Standalone {
 295             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 296             target: run.target,
 297         });
 298     }
 299
 300     /// Generates all standalone documentation as compiled by the rustdoc in `stage`
 301     /// for the `target` into `out`.
 302     ///
 303     /// This will list all of `src/doc` looking for markdown files and appropriately
 304     /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and
 305     /// `STAMP` along with providing the various header/footer HTML we've customized.
 306     ///
 307     /// In the end, this is just a glorified wrapper around rustdoc!
 308     fn run(self, builder: &Builder<'_>) {
 309         let target = self.target;
 310         let compiler = self.compiler;
 311         builder.info(&format!("Documenting standalone ({})", target));
 312         let out = builder.doc_out(target);
 313         t!(fs::create_dir_all(&out));
 314
 315         let version_info = builder.ensure(SharedAssets { target: self.target }).version_info;
 316
 317         let favicon = builder.src.join("src/doc/favicon.inc");
 318         let footer = builder.src.join("src/doc/footer.inc");
 319         let full_toc = builder.src.join("src/doc/full-toc.inc");
 320
 321         for file in t!(fs::read_dir(builder.src.join("src/doc"))) {
 322             let file = t!(file);
 323             let path = file.path();
 324             let filename = path.file_name().unwrap().to_str().unwrap();
 325             if !filename.ends_with(".md") || filename == "README.md" {
 326                 continue;
 327             }
 328
 329             let html = out.join(filename).with_extension("html");
 330             let rustdoc = builder.rustdoc(compiler);
 331             if up_to_date(&path, &html)
 332                 && up_to_date(&footer, &html)
 333                 && up_to_date(&favicon, &html)
 334                 && up_to_date(&full_toc, &html)
 335                 && (builder.config.dry_run() || up_to_date(&version_info, &html))
 336                 && (builder.config.dry_run() || up_to_date(&rustdoc, &html))
 337             {
 338                 continue;
 339             }
 340
 341             let mut cmd = builder.rustdoc_cmd(compiler);
 342             // Needed for --index-page flag
 343             cmd.arg("-Z").arg("unstable-options");
 344
 345             cmd.arg("--html-after-content")
 346                 .arg(&footer)
 347                 .arg("--html-before-content")
 348                 .arg(&version_info)
 349                 .arg("--html-in-header")
 350                 .arg(&favicon)
 351                 .arg("--markdown-no-toc")
 352                 .arg("--index-page")
 353                 .arg(&builder.src.join("src/doc/index.md"))
 354                 .arg("--markdown-playground-url")
 355                 .arg("https://play.rust-lang.org/")
 356                 .arg("-o")
 357                 .arg(&out)
 358                 .arg(&path);
 359
 360             if !builder.config.docs_minification {
 361                 cmd.arg("--disable-minification");
 362             }
 363
 364             if filename == "not_found.md" {
 365                 cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css");
 366             } else {
 367                 cmd.arg("--markdown-css").arg("rust.css");
 368             }
 369             builder.run(&mut cmd);
 370         }
 371
 372         // We open doc/index.html as the default if invoked as `x.py doc --open`
 373         // with no particular explicit doc requested (e.g. library/core).
 374         if builder.paths.is_empty() || builder.was_invoked_explicitly::<Self>(Kind::Doc) {
 375             let index = out.join("index.html");
 376             builder.open_in_browser(&index);
 377         }
 378     }
 379 }
 380
 381 #[derive(Debug, Clone)]
 382 pub struct SharedAssetsPaths {
 383     pub version_info: PathBuf,
 384 }
 385
 386 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 387 pub struct SharedAssets {
 388     target: TargetSelection,
 389 }
 390
 391 impl Step for SharedAssets {
 392     type Output = SharedAssetsPaths;
 393     const DEFAULT: bool = false;
 394
 395     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 396         // Other tasks depend on this, no need to execute it on its own
 397         run.never()
 398     }
 399
 400     // Generate shared resources used by other pieces of documentation.
 401     fn run(self, builder: &Builder<'_>) -> Self::Output {
 402         let out = builder.doc_out(self.target);
 403
 404         let version_input = builder.src.join("src").join("doc").join("version_info.html.template");
 405         let version_info = out.join("version_info.html");
 406         if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) {
 407             let info = t!(fs::read_to_string(&version_input))
 408                 .replace("VERSION", &builder.rust_release())
 409                 .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or(""))
 410                 .replace("STAMP", builder.rust_info().sha().unwrap_or(""));
 411             t!(fs::write(&version_info, &info));
 412         }
 413
 414         builder.copy(&builder.src.join("src").join("doc").join("rust.css"), &out.join("rust.css"));
 415
 416         SharedAssetsPaths { version_info }
 417     }
 418 }
 419
 420 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 421 pub struct Std {
 422     pub stage: u32,
 423     pub target: TargetSelection,
 424     pub format: DocumentationFormat,
 425 }
 426
 427 impl Step for Std {
 428     type Output = ();
 429     const DEFAULT: bool = true;
 430
 431     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 432         let builder = run.builder;
 433         run.all_krates("test").path("library").default_condition(builder.config.docs)
 434     }
 435
 436     fn make_run(run: RunConfig<'_>) {
 437         run.builder.ensure(Std {
 438             stage: run.builder.top_stage,
 439             target: run.target,
 440             format: if run.builder.config.cmd.json() {
 441                 DocumentationFormat::JSON
 442             } else {
 443                 DocumentationFormat::HTML
 444             },
 445         });
 446     }
 447
 448     /// Compile all standard library documentation.
 449     ///
 450     /// This will generate all documentation for the standard library and its
 451     /// dependencies. This is largely just a wrapper around `cargo doc`.
 452     fn run(self, builder: &Builder<'_>) {
 453         let stage = self.stage;
 454         let target = self.target;
 455         let out = match self.format {
 456             DocumentationFormat::HTML => builder.doc_out(target),
 457             DocumentationFormat::JSON => builder.json_doc_out(target),
 458         };
 459
 460         t!(fs::create_dir_all(&out));
 461
 462         if self.format == DocumentationFormat::HTML {
 463             builder.ensure(SharedAssets { target: self.target });
 464         }
 465
 466         let index_page = builder.src.join("src/doc/index.md").into_os_string();
 467         let mut extra_args = match self.format {
 468             DocumentationFormat::HTML => vec![
 469                 OsStr::new("--markdown-css"),
 470                 OsStr::new("rust.css"),
 471                 OsStr::new("--markdown-no-toc"),
 472                 OsStr::new("--index-page"),
 473                 &index_page,
 474             ],
 475             DocumentationFormat::JSON => vec![OsStr::new("--output-format"), OsStr::new("json")],
 476         };
 477
 478         if !builder.config.docs_minification {
 479             extra_args.push(OsStr::new("--disable-minification"));
 480         }
 481
 482         let requested_crates = builder
 483             .paths
 484             .iter()
 485             .map(components_simplified)
 486             .filter_map(|path| {
 487                 if path.len() >= 2 && path.get(0) == Some(&"library") {
 488                     // single crate
 489                     Some(path[1].to_owned())
 490                 } else if !path.is_empty() {
 491                     // ??
 492                     Some(path[0].to_owned())
 493                 } else {
 494                     // all library crates
 495                     None
 496                 }
 497             })
 498             .collect::<Vec<_>>();
 499
 500         doc_std(builder, self.format, stage, target, &out, &extra_args, &requested_crates);
 501
 502         // Don't open if the format is json
 503         if let DocumentationFormat::JSON = self.format {
 504             return;
 505         }
 506
 507         // Look for library/std, library/core etc in the `x.py doc` arguments and
 508         // open the corresponding rendered docs.
 509         for requested_crate in requested_crates {
 510             if requested_crate == "library" {
 511                 // For `x.py doc library --open`, open `std` by default.
 512                 let index = out.join("std").join("index.html");
 513                 builder.open_in_browser(index);
 514             } else if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) {
 515                 let index = out.join(requested_crate).join("index.html");
 516                 builder.open_in_browser(index);
 517             }
 518         }
 519     }
 520 }
 521
 522 /// Name of the crates that are visible to consumers of the standard library.
 523 /// Documentation for internal crates is handled by the rustc step, so internal crates will show
 524 /// up there.
 525 ///
 526 /// Order here is important!
 527 /// Crates need to be processed starting from the leaves, otherwise rustdoc will not
 528 /// create correct links between crates because rustdoc depends on the
 529 /// existence of the output directories to know if it should be a local
 530 /// or remote link.
 531 const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"];
 532
 533 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 534 pub enum DocumentationFormat {
 535     HTML,
 536     JSON,
 537 }
 538
 539 impl DocumentationFormat {
 540     fn as_str(&self) -> &str {
 541         match self {
 542             DocumentationFormat::HTML => "HTML",
 543             DocumentationFormat::JSON => "JSON",
 544         }
 545     }
 546 }
 547
 548 /// Build the documentation for public standard library crates.
 549 ///
 550 /// `requested_crates` can be used to build only a subset of the crates. If empty, all crates will
 551 /// be built.
 552 fn doc_std(
 553     builder: &Builder<'_>,
 554     format: DocumentationFormat,
 555     stage: u32,
 556     target: TargetSelection,
 557     out: &Path,
 558     extra_args: &[&OsStr],
 559     requested_crates: &[String],
 560 ) {
 561     builder.info(&format!(
 562         "Documenting{} stage{} library ({}) in {} format",
 563         crate_description(requested_crates),
 564         stage,
 565         target,
 566         format.as_str()
 567     ));
 568     if builder.no_std(target) == Some(true) {
 569         panic!(
 570             "building std documentation for no_std target {target} is not supported\n\
 571              Set `docs = false` in the config to disable documentation."
 572         );
 573     }
 574     let compiler = builder.compiler(stage, builder.config.build);
 575
 576     let target_doc_dir_name = if format == DocumentationFormat::JSON { "json-doc" } else { "doc" };
 577     let target_dir =
 578         builder.stage_out(compiler, Mode::Std).join(target.triple).join(target_doc_dir_name);
 579
 580     // This is directory where the compiler will place the output of the command.
 581     // We will then copy the files from this directory into the final `out` directory, the specified
 582     // as a function parameter.
 583     let out_dir = target_dir.join(target.triple).join("doc");
 584
 585     let run_cargo_rustdoc_for = |package: &str| {
 586         let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc");
 587         compile::std_cargo(builder, target, compiler.stage, &mut cargo);
 588         cargo
 589             .arg("--target-dir")
 590             .arg(&*target_dir.to_string_lossy())
 591             .arg("-p")
 592             .arg(package)
 593             .arg("-Zskip-rustdoc-fingerprint")
 594             .arg("--")
 595             .arg("-Z")
 596             .arg("unstable-options")
 597             .arg("--resource-suffix")
 598             .arg(&builder.version)
 599             .args(extra_args);
 600         if builder.config.library_docs_private_items {
 601             cargo.arg("--document-private-items").arg("--document-hidden-items");
 602         }
 603         builder.run(&mut cargo.into());
 604     };
 605
 606     for krate in STD_PUBLIC_CRATES {
 607         run_cargo_rustdoc_for(krate);
 608         if requested_crates.iter().any(|p| p == krate) {
 609             // No need to document more of the libraries if we have the one we want.
 610             break;
 611         }
 612     }
 613
 614     builder.cp_r(&out_dir, &out);
 615 }
 616
 617 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 618 pub struct Rustc {
 619     pub stage: u32,
 620     pub target: TargetSelection,
 621 }
 622
 623 impl Step for Rustc {
 624     type Output = ();
 625     const DEFAULT: bool = true;
 626     const ONLY_HOSTS: bool = true;
 627
 628     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 629         let builder = run.builder;
 630         run.crate_or_deps("rustc-main")
 631             .path("compiler")
 632             .default_condition(builder.config.compiler_docs)
 633     }
 634
 635     fn make_run(run: RunConfig<'_>) {
 636         run.builder.ensure(Rustc { stage: run.builder.top_stage, target: run.target });
 637     }
 638
 639     /// Generates compiler documentation.
 640     ///
 641     /// This will generate all documentation for compiler and dependencies.
 642     /// Compiler documentation is distributed separately, so we make sure
 643     /// we do not merge it with the other documentation from std, test and
 644     /// proc_macros. This is largely just a wrapper around `cargo doc`.
 645     fn run(self, builder: &Builder<'_>) {
 646         let stage = self.stage;
 647         let target = self.target;
 648
 649         let paths = builder
 650             .paths
 651             .iter()
 652             .filter(|path| {
 653                 let components = components_simplified(path);
 654                 components.len() >= 2 && components[0] == "compiler"
 655             })
 656             .collect::<Vec<_>>();
 657
 658         // This is the intended out directory for compiler documentation.
 659         let out = builder.compiler_doc_out(target);
 660         t!(fs::create_dir_all(&out));
 661
 662         // Build the standard library, so that proc-macros can use it.
 663         // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.)
 664         let compiler = builder.compiler(stage, builder.config.build);
 665         builder.ensure(compile::Std::new(compiler, builder.config.build));
 666
 667         builder.info(&format!("Documenting stage{} compiler ({})", stage, target));
 668
 669         // This uses a shared directory so that librustdoc documentation gets
 670         // correctly built and merged with the rustc documentation. This is
 671         // needed because rustdoc is built in a different directory from
 672         // rustc. rustdoc needs to be able to see everything, for example when
 673         // merging the search index, or generating local (relative) links.
 674         let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc");
 675         t!(symlink_dir_force(&builder.config, &out, &out_dir));
 676         // Cargo puts proc macros in `target/doc` even if you pass `--target`
 677         // explicitly (https://github.com/rust-lang/cargo/issues/7677).
 678         let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc");
 679         t!(symlink_dir_force(&builder.config, &out, &proc_macro_out_dir));
 680
 681         // Build cargo command.
 682         let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc");
 683         cargo.rustdocflag("--document-private-items");
 684         // Since we always pass --document-private-items, there's no need to warn about linking to private items.
 685         cargo.rustdocflag("-Arustdoc::private-intra-doc-links");
 686         cargo.rustdocflag("--enable-index-page");
 687         cargo.rustdocflag("-Zunstable-options");
 688         cargo.rustdocflag("-Znormalize-docs");
 689         cargo.rustdocflag("--show-type-layout");
 690         cargo.rustdocflag("--generate-link-to-definition");
 691         compile::rustc_cargo(builder, &mut cargo, target);
 692         cargo.arg("-Zunstable-options");
 693         cargo.arg("-Zskip-rustdoc-fingerprint");
 694
 695         // Only include compiler crates, no dependencies of those, such as `libc`.
 696         // Do link to dependencies on `docs.rs` however using `rustdoc-map`.
 697         cargo.arg("--no-deps");
 698         cargo.arg("-Zrustdoc-map");
 699
 700         // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies,
 701         // once this is no longer an issue the special case for `ena` can be removed.
 702         cargo.rustdocflag("--extern-html-root-url");
 703         cargo.rustdocflag("ena=https://docs.rs/ena/latest/");
 704
 705         let root_crates = if paths.is_empty() {
 706             vec![
 707                 INTERNER.intern_str("rustc_driver"),
 708                 INTERNER.intern_str("rustc_codegen_llvm"),
 709                 INTERNER.intern_str("rustc_codegen_ssa"),
 710             ]
 711         } else {
 712             paths.into_iter().map(|p| builder.crate_paths[p]).collect()
 713         };
 714         // Find dependencies for top level crates.
 715         let compiler_crates = root_crates.iter().flat_map(|krate| {
 716             builder.in_tree_crates(krate, Some(target)).into_iter().map(|krate| krate.name)
 717         });
 718
 719         let mut to_open = None;
 720         for krate in compiler_crates {
 721             // Create all crate output directories first to make sure rustdoc uses
 722             // relative links.
 723             // FIXME: Cargo should probably do this itself.
 724             t!(fs::create_dir_all(out_dir.join(krate)));
 725             cargo.arg("-p").arg(krate);
 726             if to_open.is_none() {
 727                 to_open = Some(krate);
 728             }
 729         }
 730
 731         builder.run(&mut cargo.into());
 732         // Let's open the first crate documentation page:
 733         if let Some(krate) = to_open {
 734             let index = out.join(krate).join("index.html");
 735             builder.open_in_browser(index);
 736         }
 737     }
 738 }
 739
 740 macro_rules! tool_doc {
 741     ($tool: ident, $should_run: literal, $path: literal, $(rustc_tool = $rustc_tool:literal, )? $(in_tree = $in_tree:literal, )? [$($krate: literal),+ $(,)?] $(,)?) => {
 742         #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 743         pub struct $tool {
 744             target: TargetSelection,
 745         }
 746
 747         impl Step for $tool {
 748             type Output = ();
 749             const DEFAULT: bool = true;
 750             const ONLY_HOSTS: bool = true;
 751
 752             fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 753                 let builder = run.builder;
 754                 run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs)
 755             }
 756
 757             fn make_run(run: RunConfig<'_>) {
 758                 run.builder.ensure($tool { target: run.target });
 759             }
 760
 761             /// Generates compiler documentation.
 762             ///
 763             /// This will generate all documentation for compiler and dependencies.
 764             /// Compiler documentation is distributed separately, so we make sure
 765             /// we do not merge it with the other documentation from std, test and
 766             /// proc_macros. This is largely just a wrapper around `cargo doc`.
 767             fn run(self, builder: &Builder<'_>) {
 768                 let stage = builder.top_stage;
 769                 let target = self.target;
 770
 771                 // This is the intended out directory for compiler documentation.
 772                 let out = builder.compiler_doc_out(target);
 773                 t!(fs::create_dir_all(&out));
 774
 775                 let compiler = builder.compiler(stage, builder.config.build);
 776                 builder.ensure(compile::Std::new(compiler, target));
 777
 778                 if true $(&& $rustc_tool)? {
 779                     // Build rustc docs so that we generate relative links.
 780                     builder.ensure(Rustc { stage, target });
 781
 782                     // Rustdoc needs the rustc sysroot available to build.
 783                     // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed
 784                     // with strange errors, but only on a full bors test ...
 785                     builder.ensure(compile::Rustc::new(compiler, target));
 786                 }
 787
 788                 let source_type = if true $(&& $in_tree)? {
 789                     SourceType::InTree
 790                 } else {
 791                     SourceType::Submodule
 792                 };
 793
 794                 builder.info(
 795                     &format!(
 796                         "Documenting stage{} {} ({})",
 797                         stage,
 798                         stringify!($tool).to_lowercase(),
 799                         target,
 800                     ),
 801                 );
 802
 803                 // Symlink compiler docs to the output directory of rustdoc documentation.
 804                 let out_dirs = [
 805                     builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"),
 806                     // Cargo uses a different directory for proc macros.
 807                     builder.stage_out(compiler, Mode::ToolRustc).join("doc"),
 808                 ];
 809                 for out_dir in out_dirs {
 810                     t!(fs::create_dir_all(&out_dir));
 811                     t!(symlink_dir_force(&builder.config, &out, &out_dir));
 812                 }
 813
 814                 // Build cargo command.
 815                 let mut cargo = prepare_tool_cargo(
 816                     builder,
 817                     compiler,
 818                     Mode::ToolRustc,
 819                     target,
 820                     "doc",
 821                     $path,
 822                     source_type,
 823                     &[],
 824                 );
 825
 826                 cargo.arg("-Zskip-rustdoc-fingerprint");
 827                 // Only include compiler crates, no dependencies of those, such as `libc`.
 828                 cargo.arg("--no-deps");
 829                 $(
 830                     cargo.arg("-p").arg($krate);
 831                 )+
 832
 833                 cargo.rustdocflag("--document-private-items");
 834                 cargo.rustdocflag("--enable-index-page");
 835                 cargo.rustdocflag("--show-type-layout");
 836                 cargo.rustdocflag("--generate-link-to-definition");
 837                 cargo.rustdocflag("-Zunstable-options");
 838                 builder.run(&mut cargo.into());
 839             }
 840         }
 841     }
 842 }
 843
 844 tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", ["rustdoc", "rustdoc-json-types"],);
 845 tool_doc!(
 846     Rustfmt,
 847     "rustfmt-nightly",
 848     "src/tools/rustfmt",
 849     ["rustfmt-nightly", "rustfmt-config_proc_macro"],
 850 );
 851 tool_doc!(Clippy, "clippy", "src/tools/clippy", ["clippy_utils"]);
 852 tool_doc!(Miri, "miri", "src/tools/miri", ["miri"]);
 853 tool_doc!(
 854     Cargo,
 855     "cargo",
 856     "src/tools/cargo",
 857     rustc_tool = false,
 858     in_tree = false,
 859     [
 860         "cargo",
 861         "cargo-platform",
 862         "cargo-util",
 863         "crates-io",
 864         "cargo-test-macro",
 865         "cargo-test-support",
 866         "cargo-credential",
 867         "cargo-credential-1password",
 868         "mdman",
 869         // FIXME: this trips a license check in tidy.
 870         // "resolver-tests",
 871         // FIXME: we should probably document these, but they're different per-platform so we can't use `tool_doc`.
 872         // "cargo-credential-gnome-secret",
 873         // "cargo-credential-macos-keychain",
 874         // "cargo-credential-wincred",
 875     ]
 876 );
 877
 878 #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]
 879 pub struct ErrorIndex {
 880     pub target: TargetSelection,
 881 }
 882
 883 impl Step for ErrorIndex {
 884     type Output = ();
 885     const DEFAULT: bool = true;
 886     const ONLY_HOSTS: bool = true;
 887
 888     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 889         let builder = run.builder;
 890         run.path("src/tools/error_index_generator").default_condition(builder.config.docs)
 891     }
 892
 893     fn make_run(run: RunConfig<'_>) {
 894         let target = run.target;
 895         run.builder.ensure(ErrorIndex { target });
 896     }
 897
 898     /// Generates the HTML rendered error-index by running the
 899     /// `error_index_generator` tool.
 900     fn run(self, builder: &Builder<'_>) {
 901         builder.info(&format!("Documenting error index ({})", self.target));
 902         let out = builder.doc_out(self.target);
 903         t!(fs::create_dir_all(&out));
 904         let mut index = tool::ErrorIndex::command(builder);
 905         index.arg("html");
 906         index.arg(out);
 907         index.arg(&builder.version);
 908
 909         builder.run(&mut index);
 910     }
 911 }
 912
 913 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 914 pub struct UnstableBookGen {
 915     target: TargetSelection,
 916 }
 917
 918 impl Step for UnstableBookGen {
 919     type Output = ();
 920     const DEFAULT: bool = true;
 921     const ONLY_HOSTS: bool = true;
 922
 923     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 924         let builder = run.builder;
 925         run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs)
 926     }
 927
 928     fn make_run(run: RunConfig<'_>) {
 929         run.builder.ensure(UnstableBookGen { target: run.target });
 930     }
 931
 932     fn run(self, builder: &Builder<'_>) {
 933         let target = self.target;
 934
 935         builder.info(&format!("Generating unstable book md files ({})", target));
 936         let out = builder.md_doc_out(target).join("unstable-book");
 937         builder.create_dir(&out);
 938         builder.remove_dir(&out);
 939         let mut cmd = builder.tool_cmd(Tool::UnstableBookGen);
 940         cmd.arg(builder.src.join("library"));
 941         cmd.arg(builder.src.join("compiler"));
 942         cmd.arg(builder.src.join("src"));
 943         cmd.arg(out);
 944
 945         builder.run(&mut cmd);
 946     }
 947 }
 948
 949 fn symlink_dir_force(config: &Config, src: &Path, dst: &Path) -> io::Result<()> {
 950     if config.dry_run() {
 951         return Ok(());
 952     }
 953     if let Ok(m) = fs::symlink_metadata(dst) {
 954         if m.file_type().is_dir() {
 955             fs::remove_dir_all(dst)?;
 956         } else {
 957             // handle directory junctions on windows by falling back to
 958             // `remove_dir`.
 959             fs::remove_file(dst).or_else(|_| fs::remove_dir(dst))?;
 960         }
 961     }
 962
 963     symlink_dir(config, src, dst)
 964 }
 965
 966 #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]
 967 pub struct RustcBook {
 968     pub compiler: Compiler,
 969     pub target: TargetSelection,
 970     pub validate: bool,
 971 }
 972
 973 impl Step for RustcBook {
 974     type Output = ();
 975     const DEFAULT: bool = true;
 976     const ONLY_HOSTS: bool = true;
 977
 978     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 979         let builder = run.builder;
 980         run.path("src/doc/rustc").default_condition(builder.config.docs)
 981     }
 982
 983     fn make_run(run: RunConfig<'_>) {
 984         run.builder.ensure(RustcBook {
 985             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 986             target: run.target,
 987             validate: false,
 988         });
 989     }
 990
 991     /// Builds the rustc book.
 992     ///
 993     /// The lints are auto-generated by a tool, and then merged into the book
 994     /// in the "md-doc" directory in the build output directory. Then
 995     /// "rustbook" is used to convert it to HTML.
 996     fn run(self, builder: &Builder<'_>) {
 997         let out_base = builder.md_doc_out(self.target).join("rustc");
 998         t!(fs::create_dir_all(&out_base));
 999         let out_listing = out_base.join("src/lints");
1000         builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base);
1001         builder.info(&format!("Generating lint docs ({})", self.target));
1002
1003         let rustc = builder.rustc(self.compiler);
1004         // The tool runs `rustc` for extracting output examples, so it needs a
1005         // functional sysroot.
1006         builder.ensure(compile::Std::new(self.compiler, self.target));
1007         let mut cmd = builder.tool_cmd(Tool::LintDocs);
1008         cmd.arg("--src");
1009         cmd.arg(builder.src.join("compiler"));
1010         cmd.arg("--out");
1011         cmd.arg(&out_listing);
1012         cmd.arg("--rustc");
1013         cmd.arg(&rustc);
1014         cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg());
1015         if builder.is_verbose() {
1016             cmd.arg("--verbose");
1017         }
1018         if self.validate {
1019             cmd.arg("--validate");
1020         }
1021         if !builder.unstable_features() {
1022             // We need to validate nightly features, even on the stable channel.
1023             cmd.env("RUSTC_BOOTSTRAP", "1");
1024         }
1025         // If the lib directories are in an unusual location (changed in
1026         // config.toml), then this needs to explicitly update the dylib search
1027         // path.
1028         builder.add_rustc_lib_path(self.compiler, &mut cmd);
1029         builder.run(&mut cmd);
1030         // Run rustbook/mdbook to generate the HTML pages.
1031         builder.ensure(RustbookSrc {
1032             target: self.target,
1033             name: INTERNER.intern_str("rustc"),
1034             src: INTERNER.intern_path(out_base),
1035         });
1036
1037         let out = builder.doc_out(self.target);
1038         let index = out.join("rustc").join("index.html");
1039         builder.maybe_open_in_browser::<Self>(index);
1040     }
1041 }