src/ci/stage-build.py

   1 #!/usr/bin/env python3
   2 # ignore-tidy-linelength
   3
   4 # Compatible with Python 3.6+
   5
   6 import contextlib
   7 import getpass
   8 import glob
   9 import logging
  10 import os
  11 import pprint
  12 import shutil
  13 import subprocess
  14 import sys
  15 import time
  16 import traceback
  17 import urllib.request
  18 from io import StringIO
  19 from pathlib import Path
  20 from typing import Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union
  21
  22 PGO_HOST = os.environ["PGO_HOST"]
  23
  24 LOGGER = logging.getLogger("stage-build")
  25
  26 LLVM_PGO_CRATES = [
  27     "syn-1.0.89",
  28     "cargo-0.60.0",
  29     "serde-1.0.136",
  30     "ripgrep-13.0.0",
  31     "regex-1.5.5",
  32     "clap-3.1.6",
  33     "hyper-0.14.18"
  34 ]
  35
  36 RUSTC_PGO_CRATES = [
  37     "externs",
  38     "ctfe-stress-5",
  39     "cargo-0.60.0",
  40     "token-stream-stress",
  41     "match-stress",
  42     "tuple-stress",
  43     "diesel-1.4.8",
  44     "bitmaps-3.1.0"
  45 ]
  46
  47 LLVM_BOLT_CRATES = LLVM_PGO_CRATES
  48
  49
  50 class Pipeline:
  51     # Paths
  52     def checkout_path(self) -> Path:
  53         """
  54         The root checkout, where the source is located.
  55         """
  56         raise NotImplementedError
  57
  58     def downloaded_llvm_dir(self) -> Path:
  59         """
  60         Directory where the host LLVM is located.
  61         """
  62         raise NotImplementedError
  63
  64     def build_root(self) -> Path:
  65         """
  66         The main directory where the build occurs.
  67         """
  68         raise NotImplementedError
  69
  70     def build_artifacts(self) -> Path:
  71         return self.build_root() / "build" / PGO_HOST
  72
  73     def rustc_stage_0(self) -> Path:
  74         return self.build_artifacts() / "stage0" / "bin" / "rustc"
  75
  76     def cargo_stage_0(self) -> Path:
  77         return self.build_artifacts() / "stage0" / "bin" / "cargo"
  78
  79     def rustc_stage_2(self) -> Path:
  80         return self.build_artifacts() / "stage2" / "bin" / "rustc"
  81
  82     def opt_artifacts(self) -> Path:
  83         raise NotImplementedError
  84
  85     def llvm_profile_dir_root(self) -> Path:
  86         return self.opt_artifacts() / "llvm-pgo"
  87
  88     def llvm_profile_merged_file(self) -> Path:
  89         return self.opt_artifacts() / "llvm-pgo.profdata"
  90
  91     def rustc_perf_dir(self) -> Path:
  92         return self.opt_artifacts() / "rustc-perf"
  93
  94     def build_rustc_perf(self):
  95         raise NotImplementedError()
  96
  97     def rustc_profile_dir_root(self) -> Path:
  98         return self.opt_artifacts() / "rustc-pgo"
  99
 100     def rustc_profile_merged_file(self) -> Path:
 101         return self.opt_artifacts() / "rustc-pgo.profdata"
 102
 103     def rustc_profile_template_path(self) -> Path:
 104         """
 105         The profile data is written into a single filepath that is being repeatedly merged when each
 106         rustc invocation ends. Empirically, this can result in some profiling data being lost. That's
 107         why we override the profile path to include the PID. This will produce many more profiling
 108         files, but the resulting profile will produce a slightly faster rustc binary.
 109         """
 110         return self.rustc_profile_dir_root() / "default_%m_%p.profraw"
 111
 112     def supports_bolt(self) -> bool:
 113         raise NotImplementedError
 114
 115     def llvm_bolt_profile_merged_file(self) -> Path:
 116         return self.opt_artifacts() / "bolt.profdata"
 117
 118
 119 class LinuxPipeline(Pipeline):
 120     def checkout_path(self) -> Path:
 121         return Path("/checkout")
 122
 123     def downloaded_llvm_dir(self) -> Path:
 124         return Path("/rustroot")
 125
 126     def build_root(self) -> Path:
 127         return self.checkout_path() / "obj"
 128
 129     def opt_artifacts(self) -> Path:
 130         return Path("/tmp/tmp-multistage/opt-artifacts")
 131
 132     def build_rustc_perf(self):
 133         # /tmp/rustc-perf comes from the Dockerfile
 134         shutil.copytree("/tmp/rustc-perf", self.rustc_perf_dir())
 135         cmd(["chown", "-R", f"{getpass.getuser()}:", self.rustc_perf_dir()])
 136
 137         with change_cwd(self.rustc_perf_dir()):
 138             cmd([self.cargo_stage_0(), "build", "-p", "collector"], env=dict(
 139                 RUSTC=str(self.rustc_stage_0()),
 140                 RUSTC_BOOTSTRAP="1"
 141             ))
 142
 143     def supports_bolt(self) -> bool:
 144         return True
 145
 146
 147 class WindowsPipeline(Pipeline):
 148     def __init__(self):
 149         self.checkout_dir = Path(os.getcwd())
 150
 151     def checkout_path(self) -> Path:
 152         return self.checkout_dir
 153
 154     def downloaded_llvm_dir(self) -> Path:
 155         return self.checkout_path() / "citools" / "clang-rust"
 156
 157     def build_root(self) -> Path:
 158         return self.checkout_path()
 159
 160     def opt_artifacts(self) -> Path:
 161         return self.checkout_path() / "opt-artifacts"
 162
 163     def rustc_stage_0(self) -> Path:
 164         return super().rustc_stage_0().with_suffix(".exe")
 165
 166     def cargo_stage_0(self) -> Path:
 167         return super().cargo_stage_0().with_suffix(".exe")
 168
 169     def rustc_stage_2(self) -> Path:
 170         return super().rustc_stage_2().with_suffix(".exe")
 171
 172     def build_rustc_perf(self):
 173         # rustc-perf version from 2022-07-22
 174         perf_commit = "3c253134664fdcba862c539d37f0de18557a9a4c"
 175         rustc_perf_zip_path = self.opt_artifacts() / "perf.zip"
 176
 177         def download_rustc_perf():
 178             download_file(
 179                 f"https://github.com/rust-lang/rustc-perf/archive/{perf_commit}.zip",
 180                 rustc_perf_zip_path
 181             )
 182             with change_cwd(self.opt_artifacts()):
 183                 unpack_archive(rustc_perf_zip_path)
 184                 move_path(Path(f"rustc-perf-{perf_commit}"), self.rustc_perf_dir())
 185                 delete_file(rustc_perf_zip_path)
 186
 187         retry_action(download_rustc_perf, "Download rustc-perf")
 188
 189         with change_cwd(self.rustc_perf_dir()):
 190             cmd([self.cargo_stage_0(), "build", "-p", "collector"], env=dict(
 191                 RUSTC=str(self.rustc_stage_0()),
 192                 RUSTC_BOOTSTRAP="1"
 193             ))
 194
 195     def rustc_profile_template_path(self) -> Path:
 196         """
 197         On Windows, we don't have enough space to use separate files for each rustc invocation.
 198         Therefore, we use a single file for the generated profiles.
 199         """
 200         return self.rustc_profile_dir_root() / "default_%m.profraw"
 201
 202     def supports_bolt(self) -> bool:
 203         return False
 204
 205
 206 def get_timestamp() -> float:
 207     return time.time()
 208
 209
 210 Duration = float
 211 TimerSection = Union[Duration, "Timer"]
 212
 213
 214 def iterate_sections(section: TimerSection, name: str, level: int = 0) -> Iterator[Tuple[int, str, Duration]]:
 215     """
 216     Hierarchically iterate the sections of a timer, in a depth-first order.
 217     """
 218     if isinstance(section, Duration):
 219         yield (level, name, section)
 220     elif isinstance(section, Timer):
 221         yield (level, name, section.total_duration())
 222         for (child_name, child_section) in section.sections:
 223             yield from iterate_sections(child_section, child_name, level=level + 1)
 224     else:
 225         assert False
 226
 227
 228 class Timer:
 229     def __init__(self, parent_names: Tuple[str, ...] = ()):
 230         self.sections: List[Tuple[str, TimerSection]] = []
 231         self.section_active = False
 232         self.parent_names = parent_names
 233
 234     @contextlib.contextmanager
 235     def section(self, name: str) -> "Timer":
 236         assert not self.section_active
 237         self.section_active = True
 238
 239         start = get_timestamp()
 240         exc = None
 241
 242         child_timer = Timer(parent_names=self.parent_names + (name, ))
 243         full_name = " > ".join(child_timer.parent_names)
 244         try:
 245             LOGGER.info(f"Section `{full_name}` starts")
 246             yield child_timer
 247         except BaseException as exception:
 248             exc = exception
 249             raise
 250         finally:
 251             end = get_timestamp()
 252             duration = end - start
 253
 254             if child_timer.has_children():
 255                 self.sections.append((name, child_timer))
 256             else:
 257                 self.sections.append((name, duration))
 258             if exc is None:
 259                 LOGGER.info(f"Section `{full_name}` ended: OK ({duration:.2f}s)")
 260             else:
 261                 LOGGER.info(f"Section `{full_name}` ended: FAIL ({duration:.2f}s)")
 262             self.section_active = False
 263
 264     def total_duration(self) -> Duration:
 265         duration = 0
 266         for (_, section) in self.sections:
 267             if isinstance(section, Duration):
 268                 duration += section
 269             else:
 270                 duration += section.total_duration()
 271         return duration
 272
 273     def has_children(self) -> bool:
 274         return len(self.sections) > 0
 275
 276     def print_stats(self):
 277         rows = []
 278         for (child_name, child_section) in self.sections:
 279             for (level, name, duration) in iterate_sections(child_section, child_name, level=0):
 280                 label = f"{' ' * level}{name}:"
 281                 rows.append((label, duration))
 282
 283         # Empty row
 284         rows.append(("", ""))
 285
 286         total_duration_label = "Total duration:"
 287         total_duration = self.total_duration()
 288         rows.append((total_duration_label, humantime(total_duration)))
 289
 290         space_after_label = 2
 291         max_label_length = max(16, max(len(label) for (label, _) in rows)) + space_after_label
 292
 293         table_width = max_label_length + 23
 294         divider = "-" * table_width
 295
 296         with StringIO() as output:
 297             print(divider, file=output)
 298             for (label, duration) in rows:
 299                 if isinstance(duration, Duration):
 300                     pct = (duration / total_duration) * 100
 301                     value = f"{duration:>12.2f}s ({pct:>5.2f}%)"
 302                 else:
 303                     value = f"{duration:>{len(total_duration_label) + 7}}"
 304                 print(f"{label:<{max_label_length}} {value}", file=output)
 305             print(divider, file=output, end="")
 306             LOGGER.info(f"Timer results\n{output.getvalue()}")
 307
 308
 309 @contextlib.contextmanager
 310 def change_cwd(dir: Path):
 311     """
 312     Temporarily change working directory to `dir`.
 313     """
 314     cwd = os.getcwd()
 315     LOGGER.debug(f"Changing working dir from `{cwd}` to `{dir}`")
 316     os.chdir(dir)
 317     try:
 318         yield
 319     finally:
 320         LOGGER.debug(f"Reverting working dir to `{cwd}`")
 321         os.chdir(cwd)
 322
 323
 324 def humantime(time_s: float) -> str:
 325     hours = time_s // 3600
 326     time_s = time_s % 3600
 327     minutes = time_s // 60
 328     seconds = time_s % 60
 329
 330     result = ""
 331     if hours > 0:
 332         result += f"{int(hours)}h "
 333     if minutes > 0:
 334         result += f"{int(minutes)}m "
 335     result += f"{round(seconds)}s"
 336     return result
 337
 338
 339 def move_path(src: Path, dst: Path):
 340     LOGGER.info(f"Moving `{src}` to `{dst}`")
 341     shutil.move(src, dst)
 342
 343
 344 def delete_file(path: Path):
 345     LOGGER.info(f"Deleting file `{path}`")
 346     os.unlink(path)
 347
 348
 349 def delete_directory(path: Path):
 350     LOGGER.info(f"Deleting directory `{path}`")
 351     shutil.rmtree(path)
 352
 353
 354 def unpack_archive(archive: Path):
 355     LOGGER.info(f"Unpacking archive `{archive}`")
 356     shutil.unpack_archive(archive)
 357
 358
 359 def download_file(src: str, target: Path):
 360     LOGGER.info(f"Downloading `{src}` into `{target}`")
 361     urllib.request.urlretrieve(src, str(target))
 362
 363
 364 def retry_action(action, name: str, max_fails: int = 5):
 365     LOGGER.info(f"Attempting to perform action `{name}` with retry")
 366     for iteration in range(max_fails):
 367         LOGGER.info(f"Attempt {iteration + 1}/{max_fails}")
 368         try:
 369             action()
 370             return
 371         except:
 372             LOGGER.error(f"Action `{name}` has failed\n{traceback.format_exc()}")
 373
 374     raise Exception(f"Action `{name}` has failed after {max_fails} attempts")
 375
 376
 377 def cmd(
 378         args: List[Union[str, Path]],
 379         env: Optional[Dict[str, str]] = None,
 380         output_path: Optional[Path] = None
 381 ):
 382     args = [str(arg) for arg in args]
 383
 384     environment = os.environ.copy()
 385
 386     cmd_str = ""
 387     if env is not None:
 388         environment.update(env)
 389         cmd_str += " ".join(f"{k}={v}" for (k, v) in (env or {}).items())
 390         cmd_str += " "
 391     cmd_str += " ".join(args)
 392     if output_path is not None:
 393         cmd_str += f" > {output_path}"
 394     LOGGER.info(f"Executing `{cmd_str}`")
 395
 396     if output_path is not None:
 397         with open(output_path, "w") as f:
 398             return subprocess.run(
 399                 args,
 400                 env=environment,
 401                 check=True,
 402                 stdout=f
 403             )
 404     return subprocess.run(args, env=environment, check=True)
 405
 406
 407 def run_compiler_benchmarks(
 408         pipeline: Pipeline,
 409         profiles: List[str],
 410         scenarios: List[str],
 411         crates: List[str],
 412         env: Optional[Dict[str, str]] = None
 413 ):
 414     env = env if env is not None else {}
 415
 416     # Compile libcore, both in opt-level=0 and opt-level=3
 417     with change_cwd(pipeline.build_root()):
 418         cmd([
 419             pipeline.rustc_stage_2(),
 420             "--edition", "2021",
 421             "--crate-type", "lib",
 422             str(pipeline.checkout_path() / "library/core/src/lib.rs"),
 423             "--out-dir", pipeline.opt_artifacts()
 424         ], env=dict(RUSTC_BOOTSTRAP="1", **env))
 425
 426         cmd([
 427             pipeline.rustc_stage_2(),
 428             "--edition", "2021",
 429             "--crate-type", "lib",
 430             "-Copt-level=3",
 431             str(pipeline.checkout_path() / "library/core/src/lib.rs"),
 432             "--out-dir", pipeline.opt_artifacts()
 433         ], env=dict(RUSTC_BOOTSTRAP="1", **env))
 434
 435     # Run rustc-perf benchmarks
 436     # Benchmark using profile_local with eprintln, which essentially just means
 437     # don't actually benchmark -- just make sure we run rustc a bunch of times.
 438     with change_cwd(pipeline.rustc_perf_dir()):
 439         cmd([
 440             pipeline.cargo_stage_0(),
 441             "run",
 442             "-p", "collector", "--bin", "collector", "--",
 443             "profile_local", "eprintln",
 444             pipeline.rustc_stage_2(),
 445             "--id", "Test",
 446             "--cargo", pipeline.cargo_stage_0(),
 447             "--profiles", ",".join(profiles),
 448             "--scenarios", ",".join(scenarios),
 449             "--include", ",".join(crates)
 450         ], env=dict(
 451             RUST_LOG="collector=debug",
 452             RUSTC=str(pipeline.rustc_stage_0()),
 453             RUSTC_BOOTSTRAP="1",
 454             **env
 455         ))
 456
 457
 458 # https://stackoverflow.com/a/31631711/1107768
 459 def format_bytes(size: int) -> str:
 460     """Return the given bytes as a human friendly KiB, MiB or GiB string."""
 461     KB = 1024
 462     MB = KB ** 2  # 1,048,576
 463     GB = KB ** 3  # 1,073,741,824
 464     TB = KB ** 4  # 1,099,511,627,776
 465
 466     if size < KB:
 467         return f"{size} B"
 468     elif KB <= size < MB:
 469         return f"{size / KB:.2f} KiB"
 470     elif MB <= size < GB:
 471         return f"{size / MB:.2f} MiB"
 472     elif GB <= size < TB:
 473         return f"{size / GB:.2f} GiB"
 474     else:
 475         return str(size)
 476
 477
 478 # https://stackoverflow.com/a/63307131/1107768
 479 def count_files(path: Path) -> int:
 480     return sum(1 for p in path.rglob("*") if p.is_file())
 481
 482
 483 def count_files_with_prefix(path: Path) -> int:
 484     return sum(1 for p in glob.glob(f"{path}*") if Path(p).is_file())
 485
 486
 487 # https://stackoverflow.com/a/55659577/1107768
 488 def get_path_size(path: Path) -> int:
 489     if path.is_dir():
 490         return sum(p.stat().st_size for p in path.rglob("*"))
 491     return path.stat().st_size
 492
 493
 494 def get_path_prefix_size(path: Path) -> int:
 495     """
 496     Get size of all files beginning with the prefix `path`.
 497     Alternative to shell `du -sh <path>*`.
 498     """
 499     return sum(Path(p).stat().st_size for p in glob.glob(f"{path}*"))
 500
 501
 502 def get_files(directory: Path, filter: Optional[Callable[[Path], bool]] = None) -> Iterable[Path]:
 503     for file in os.listdir(directory):
 504         path = directory / file
 505         if filter is None or filter(path):
 506             yield path
 507
 508
 509 def build_rustc(
 510         pipeline: Pipeline,
 511         args: List[str],
 512         env: Optional[Dict[str, str]] = None
 513 ):
 514     arguments = [
 515                     sys.executable,
 516                     pipeline.checkout_path() / "x.py",
 517                     "build",
 518                     "--target", PGO_HOST,
 519                     "--host", PGO_HOST,
 520                     "--stage", "2",
 521                     "library/std"
 522                 ] + args
 523     cmd(arguments, env=env)
 524
 525
 526 def create_pipeline() -> Pipeline:
 527     if sys.platform == "linux":
 528         return LinuxPipeline()
 529     elif sys.platform in ("cygwin", "win32"):
 530         return WindowsPipeline()
 531     else:
 532         raise Exception(f"Optimized build is not supported for platform {sys.platform}")
 533
 534
 535 def gather_llvm_profiles(pipeline: Pipeline):
 536     LOGGER.info("Running benchmarks with PGO instrumented LLVM")
 537     run_compiler_benchmarks(
 538         pipeline,
 539         profiles=["Debug", "Opt"],
 540         scenarios=["Full"],
 541         crates=LLVM_PGO_CRATES
 542     )
 543
 544     profile_path = pipeline.llvm_profile_merged_file()
 545     LOGGER.info(f"Merging LLVM PGO profiles to {profile_path}")
 546     cmd([
 547         pipeline.downloaded_llvm_dir() / "bin" / "llvm-profdata",
 548         "merge",
 549         "-o", profile_path,
 550         pipeline.llvm_profile_dir_root()
 551     ])
 552
 553     LOGGER.info("LLVM PGO statistics")
 554     LOGGER.info(f"{profile_path}: {format_bytes(get_path_size(profile_path))}")
 555     LOGGER.info(
 556         f"{pipeline.llvm_profile_dir_root()}: {format_bytes(get_path_size(pipeline.llvm_profile_dir_root()))}")
 557     LOGGER.info(f"Profile file count: {count_files(pipeline.llvm_profile_dir_root())}")
 558
 559     # We don't need the individual .profraw files now that they have been merged
 560     # into a final .profdata
 561     delete_directory(pipeline.llvm_profile_dir_root())
 562
 563
 564 def gather_rustc_profiles(pipeline: Pipeline):
 565     LOGGER.info("Running benchmarks with PGO instrumented rustc")
 566
 567     # Here we're profiling the `rustc` frontend, so we also include `Check`.
 568     # The benchmark set includes various stress tests that put the frontend under pressure.
 569     run_compiler_benchmarks(
 570         pipeline,
 571         profiles=["Check", "Debug", "Opt"],
 572         scenarios=["All"],
 573         crates=RUSTC_PGO_CRATES,
 574         env=dict(
 575             LLVM_PROFILE_FILE=str(pipeline.rustc_profile_template_path())
 576         )
 577     )
 578
 579     profile_path = pipeline.rustc_profile_merged_file()
 580     LOGGER.info(f"Merging Rustc PGO profiles to {profile_path}")
 581     cmd([
 582         pipeline.build_artifacts() / "llvm" / "bin" / "llvm-profdata",
 583         "merge",
 584         "-o", profile_path,
 585         pipeline.rustc_profile_dir_root()
 586     ])
 587
 588     LOGGER.info("Rustc PGO statistics")
 589     LOGGER.info(f"{profile_path}: {format_bytes(get_path_size(profile_path))}")
 590     LOGGER.info(
 591         f"{pipeline.rustc_profile_dir_root()}: {format_bytes(get_path_size(pipeline.rustc_profile_dir_root()))}")
 592     LOGGER.info(f"Profile file count: {count_files(pipeline.rustc_profile_dir_root())}")
 593
 594     # We don't need the individual .profraw files now that they have been merged
 595     # into a final .profdata
 596     delete_directory(pipeline.rustc_profile_dir_root())
 597
 598
 599 def gather_llvm_bolt_profiles(pipeline: Pipeline):
 600     LOGGER.info("Running benchmarks with BOLT instrumented LLVM")
 601     run_compiler_benchmarks(
 602         pipeline,
 603         profiles=["Check", "Debug", "Opt"],
 604         scenarios=["Full"],
 605         crates=LLVM_BOLT_CRATES
 606     )
 607
 608     merged_profile_path = pipeline.llvm_bolt_profile_merged_file()
 609     profile_files_path = Path("/tmp/prof.fdata")
 610     LOGGER.info(f"Merging LLVM BOLT profiles to {merged_profile_path}")
 611
 612     profile_files = sorted(glob.glob(f"{profile_files_path}*"))
 613     cmd([
 614         "merge-fdata",
 615         *profile_files,
 616     ], output_path=merged_profile_path)
 617
 618     LOGGER.info("LLVM BOLT statistics")
 619     LOGGER.info(f"{merged_profile_path}: {format_bytes(get_path_size(merged_profile_path))}")
 620     LOGGER.info(
 621         f"{profile_files_path}: {format_bytes(get_path_prefix_size(profile_files_path))}")
 622     LOGGER.info(f"Profile file count: {count_files_with_prefix(profile_files_path)}")
 623
 624
 625 def clear_llvm_files(pipeline: Pipeline):
 626     """
 627     Rustbuild currently doesn't support rebuilding LLVM when PGO options
 628     change (or any other llvm-related options); so just clear out the relevant
 629     directories ourselves.
 630     """
 631     LOGGER.info("Clearing LLVM build files")
 632     delete_directory(pipeline.build_artifacts() / "llvm")
 633     delete_directory(pipeline.build_artifacts() / "lld")
 634
 635
 636 def print_binary_sizes(pipeline: Pipeline):
 637     bin_dir = pipeline.build_artifacts() / "stage2" / "bin"
 638     binaries = get_files(bin_dir)
 639
 640     lib_dir = pipeline.build_artifacts() / "stage2" / "lib"
 641     libraries = get_files(lib_dir, lambda p: p.suffix == ".so")
 642
 643     paths = sorted(binaries) + sorted(libraries)
 644     with StringIO() as output:
 645         for path in paths:
 646             path_str = f"{path.name}:"
 647             print(f"{path_str:<30}{format_bytes(path.stat().st_size):>14}", file=output)
 648         LOGGER.info(f"Rustc binary size\n{output.getvalue()}")
 649
 650
 651 def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: List[str]):
 652     # Clear and prepare tmp directory
 653     shutil.rmtree(pipeline.opt_artifacts(), ignore_errors=True)
 654     os.makedirs(pipeline.opt_artifacts(), exist_ok=True)
 655
 656     pipeline.build_rustc_perf()
 657
 658     # Stage 1: Build rustc + PGO instrumented LLVM
 659     with timer.section("Stage 1 (LLVM PGO)") as stage1:
 660         with stage1.section("Build rustc and LLVM"):
 661             build_rustc(pipeline, args=[
 662                 "--llvm-profile-generate"
 663             ], env=dict(
 664                 LLVM_PROFILE_DIR=str(pipeline.llvm_profile_dir_root() / "prof-%p")
 665             ))
 666
 667         with stage1.section("Gather profiles"):
 668             gather_llvm_profiles(pipeline)
 669
 670     clear_llvm_files(pipeline)
 671     final_build_args += [
 672         "--llvm-profile-use",
 673         pipeline.llvm_profile_merged_file()
 674     ]
 675
 676     # Stage 2: Build PGO instrumented rustc + LLVM
 677     with timer.section("Stage 2 (rustc PGO)") as stage2:
 678         with stage2.section("Build rustc and LLVM"):
 679             build_rustc(pipeline, args=[
 680                 "--rust-profile-generate",
 681                 pipeline.rustc_profile_dir_root()
 682             ])
 683
 684         with stage2.section("Gather profiles"):
 685             gather_rustc_profiles(pipeline)
 686
 687     clear_llvm_files(pipeline)
 688     final_build_args += [
 689         "--rust-profile-use",
 690         pipeline.rustc_profile_merged_file()
 691     ]
 692
 693     # Stage 3: Build rustc + BOLT instrumented LLVM
 694     if pipeline.supports_bolt():
 695         with timer.section("Stage 3 (LLVM BOLT)") as stage3:
 696             with stage3.section("Build rustc and LLVM"):
 697                 build_rustc(pipeline, args=[
 698                     "--llvm-profile-use",
 699                     pipeline.llvm_profile_merged_file(),
 700                     "--llvm-bolt-profile-generate",
 701                 ])
 702             with stage3.section("Gather profiles"):
 703                 gather_llvm_bolt_profiles(pipeline)
 704
 705         clear_llvm_files(pipeline)
 706         final_build_args += [
 707             "--llvm-bolt-profile-use",
 708             pipeline.llvm_bolt_profile_merged_file()
 709         ]
 710
 711     # Stage 4: Build PGO optimized rustc + PGO/BOLT optimized LLVM
 712     with timer.section("Stage 4 (final build)"):
 713         cmd(final_build_args)
 714
 715
 716 if __name__ == "__main__":
 717     logging.basicConfig(
 718         level=logging.DEBUG,
 719         format="%(name)s %(levelname)-4s: %(message)s",
 720     )
 721
 722     LOGGER.info(f"Running multi-stage build using Python {sys.version}")
 723     LOGGER.info(f"Environment values\n{pprint.pformat(dict(os.environ), indent=2)}")
 724
 725     build_args = sys.argv[1:]
 726
 727     timer = Timer()
 728     pipeline = create_pipeline()
 729     try:
 730         execute_build_pipeline(timer, pipeline, build_args)
 731     except BaseException as e:
 732         LOGGER.error("The multi-stage build has failed")
 733         raise e
 734     finally:
 735         timer.print_stats()
 736
 737     print_binary_sizes(pipeline)