]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
Auto merge of #95826 - carbotaniuman:miri-permissive-provenance, r=RalfJung
[rust.git] / src / bootstrap / bootstrap.py
1 from __future__ import absolute_import, division, print_function
2 import argparse
3 import contextlib
4 import datetime
5 import distutils.version
6 import hashlib
7 import json
8 import os
9 import re
10 import shutil
11 import subprocess
12 import sys
13 import tarfile
14 import tempfile
15
16 from time import time, sleep
17
18 # Acquire a lock on the build directory to make sure that
19 # we don't cause a race condition while building
20 # Lock is created in `build_dir/lock.db`
21 def acquire_lock(build_dir):
22     try:
23         import sqlite3
24
25         path = os.path.join(build_dir, "lock.db")
26         try:
27             con = sqlite3.Connection(path, timeout=0)
28             curs = con.cursor()
29             curs.execute("BEGIN EXCLUSIVE")
30             # The lock is released when the cursor is dropped
31             return curs
32         # If the database is busy then lock has already been acquired
33         # so we wait for the lock.
34         # We retry every quarter second so that execution is passed back to python
35         # so that it can handle signals
36         except sqlite3.OperationalError:
37             del con
38             del curs
39             print("Waiting for lock on build directory")
40             con = sqlite3.Connection(path, timeout=0.25)
41             curs = con.cursor()
42             while True:
43                 try:
44                     curs.execute("BEGIN EXCLUSIVE")
45                     break
46                 except sqlite3.OperationalError:
47                     pass
48                 sleep(0.25)
49             return curs
50     except ImportError:
51         print("warning: sqlite3 not available in python, skipping build directory lock")
52         print("please file an issue on rust-lang/rust")
53         print("this is not a problem for non-concurrent x.py invocations")
54         return None
55
56 def support_xz():
57     try:
58         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
59             temp_path = temp_file.name
60         with tarfile.open(temp_path, "w:xz"):
61             pass
62         return True
63     except tarfile.CompressionError:
64         return False
65
66 def get(base, url, path, checksums, verbose=False, do_verify=True, help_on_error=None):
67     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
68         temp_path = temp_file.name
69
70     try:
71         if do_verify:
72             if url not in checksums:
73                 raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
74                                     "Pre-built artifacts might not available for this "
75                                     "target at this time, see https://doc.rust-lang.org/nightly"
76                                     "/rustc/platform-support.html for more information.")
77                                    .format(url))
78             sha256 = checksums[url]
79             if os.path.exists(path):
80                 if verify(path, sha256, False):
81                     if verbose:
82                         print("using already-download file", path)
83                     return
84                 else:
85                     if verbose:
86                         print("ignoring already-download file",
87                             path, "due to failed verification")
88                     os.unlink(path)
89         download(temp_path, "{}/{}".format(base, url), True, verbose, help_on_error=help_on_error)
90         if do_verify and not verify(temp_path, sha256, verbose):
91             raise RuntimeError("failed verification")
92         if verbose:
93             print("moving {} to {}".format(temp_path, path))
94         shutil.move(temp_path, path)
95     finally:
96         if os.path.isfile(temp_path):
97             if verbose:
98                 print("removing", temp_path)
99             os.unlink(temp_path)
100
101
102 def download(path, url, probably_big, verbose, help_on_error=None):
103     for _ in range(0, 4):
104         try:
105             _download(path, url, probably_big, verbose, True, help_on_error=help_on_error)
106             return
107         except RuntimeError:
108             print("\nspurious failure, trying again")
109     _download(path, url, probably_big, verbose, False, help_on_error=help_on_error)
110
111
112 def _download(path, url, probably_big, verbose, exception, help_on_error=None):
113     if probably_big or verbose:
114         print("downloading {}".format(url))
115     # see https://serverfault.com/questions/301128/how-to-download
116     if sys.platform == 'win32':
117         run(["PowerShell.exe", "/nologo", "-Command",
118              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
119              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
120             verbose=verbose,
121             exception=exception)
122     else:
123         if probably_big or verbose:
124             option = "-#"
125         else:
126             option = "-s"
127         require(["curl", "--version"])
128         run(["curl", option,
129              "-L", # Follow redirect.
130              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
131              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
132              "--retry", "3", "-Sf", "-o", path, url],
133             verbose=verbose,
134             exception=exception,
135             help_on_error=help_on_error)
136
137
138 def verify(path, expected, verbose):
139     """Check if the sha256 sum of the given path is valid"""
140     if verbose:
141         print("verifying", path)
142     with open(path, "rb") as source:
143         found = hashlib.sha256(source.read()).hexdigest()
144     verified = found == expected
145     if not verified:
146         print("invalid checksum:\n"
147               "    found:    {}\n"
148               "    expected: {}".format(found, expected))
149     return verified
150
151
152 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
153     """Unpack the given tarball file"""
154     print("extracting", tarball)
155     fname = os.path.basename(tarball).replace(tarball_suffix, "")
156     with contextlib.closing(tarfile.open(tarball)) as tar:
157         for member in tar.getnames():
158             if "/" not in member:
159                 continue
160             name = member.replace(fname + "/", "", 1)
161             if match is not None and not name.startswith(match):
162                 continue
163             name = name[len(match) + 1:]
164
165             dst_path = os.path.join(dst, name)
166             if verbose:
167                 print("  extracting", member)
168             tar.extract(member, dst)
169             src_path = os.path.join(dst, member)
170             if os.path.isdir(src_path) and os.path.exists(dst_path):
171                 continue
172             shutil.move(src_path, dst_path)
173     shutil.rmtree(os.path.join(dst, fname))
174
175
176 def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error=None, **kwargs):
177     """Run a child program in a new process"""
178     if verbose:
179         print("running: " + ' '.join(args))
180     sys.stdout.flush()
181     # Use Popen here instead of call() as it apparently allows powershell on
182     # Windows to not lock up waiting for input presumably.
183     ret = subprocess.Popen(args, **kwargs)
184     code = ret.wait()
185     if code != 0:
186         err = "failed to run: " + ' '.join(args)
187         if help_on_error is not None:
188             err += "\n" + help_on_error
189         if verbose or exception:
190             raise RuntimeError(err)
191         # For most failures, we definitely do want to print this error, or the user will have no
192         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
193         # have already printed an error above, so there's no need to print the exact command we're
194         # running.
195         if is_bootstrap:
196             sys.exit(1)
197         else:
198             sys.exit(err)
199
200
201 def require(cmd, exit=True):
202     '''Run a command, returning its output.
203     On error,
204         If `exit` is `True`, exit the process.
205         Otherwise, return None.'''
206     try:
207         return subprocess.check_output(cmd).strip()
208     except (subprocess.CalledProcessError, OSError) as exc:
209         if not exit:
210             return None
211         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
212         print("Please make sure it's installed and in the path.")
213         sys.exit(1)
214
215
216 def format_build_time(duration):
217     """Return a nicer format for build time
218
219     >>> format_build_time('300')
220     '0:05:00'
221     """
222     return str(datetime.timedelta(seconds=int(duration)))
223
224
225 def default_build_triple(verbose):
226     """Build triple as in LLVM"""
227     # If the user already has a host build triple with an existing `rustc`
228     # install, use their preference. This fixes most issues with Windows builds
229     # being detected as GNU instead of MSVC.
230     default_encoding = sys.getdefaultencoding()
231     try:
232         version = subprocess.check_output(["rustc", "--version", "--verbose"],
233                 stderr=subprocess.DEVNULL)
234         version = version.decode(default_encoding)
235         host = next(x for x in version.split('\n') if x.startswith("host: "))
236         triple = host.split("host: ")[1]
237         if verbose:
238             print("detected default triple {} from pre-installed rustc".format(triple))
239         return triple
240     except Exception as e:
241         if verbose:
242             print("pre-installed rustc not detected: {}".format(e))
243             print("falling back to auto-detect")
244
245     required = sys.platform != 'win32'
246     ostype = require(["uname", "-s"], exit=required)
247     cputype = require(['uname', '-m'], exit=required)
248
249     # If we do not have `uname`, assume Windows.
250     if ostype is None or cputype is None:
251         return 'x86_64-pc-windows-msvc'
252
253     ostype = ostype.decode(default_encoding)
254     cputype = cputype.decode(default_encoding)
255
256     # The goal here is to come up with the same triple as LLVM would,
257     # at least for the subset of platforms we're willing to target.
258     ostype_mapper = {
259         'Darwin': 'apple-darwin',
260         'DragonFly': 'unknown-dragonfly',
261         'FreeBSD': 'unknown-freebsd',
262         'Haiku': 'unknown-haiku',
263         'NetBSD': 'unknown-netbsd',
264         'OpenBSD': 'unknown-openbsd'
265     }
266
267     # Consider the direct transformation first and then the special cases
268     if ostype in ostype_mapper:
269         ostype = ostype_mapper[ostype]
270     elif ostype == 'Linux':
271         os_from_sp = subprocess.check_output(
272             ['uname', '-o']).strip().decode(default_encoding)
273         if os_from_sp == 'Android':
274             ostype = 'linux-android'
275         else:
276             ostype = 'unknown-linux-gnu'
277     elif ostype == 'SunOS':
278         ostype = 'pc-solaris'
279         # On Solaris, uname -m will return a machine classification instead
280         # of a cpu type, so uname -p is recommended instead.  However, the
281         # output from that option is too generic for our purposes (it will
282         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
283         # must be used instead.
284         cputype = require(['isainfo', '-k']).decode(default_encoding)
285         # sparc cpus have sun as a target vendor
286         if 'sparc' in cputype:
287             ostype = 'sun-solaris'
288     elif ostype.startswith('MINGW'):
289         # msys' `uname` does not print gcc configuration, but prints msys
290         # configuration. so we cannot believe `uname -m`:
291         # msys1 is always i686 and msys2 is always x86_64.
292         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
293         # MINGW64 on x86_64.
294         ostype = 'pc-windows-gnu'
295         cputype = 'i686'
296         if os.environ.get('MSYSTEM') == 'MINGW64':
297             cputype = 'x86_64'
298     elif ostype.startswith('MSYS'):
299         ostype = 'pc-windows-gnu'
300     elif ostype.startswith('CYGWIN_NT'):
301         cputype = 'i686'
302         if ostype.endswith('WOW64'):
303             cputype = 'x86_64'
304         ostype = 'pc-windows-gnu'
305     elif sys.platform == 'win32':
306         # Some Windows platforms might have a `uname` command that returns a
307         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
308         # these cases, fall back to using sys.platform.
309         return 'x86_64-pc-windows-msvc'
310     else:
311         err = "unknown OS type: {}".format(ostype)
312         sys.exit(err)
313
314     if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
315         cputype = subprocess.check_output(
316               ['uname', '-p']).strip().decode(default_encoding)
317     cputype_mapper = {
318         'BePC': 'i686',
319         'aarch64': 'aarch64',
320         'amd64': 'x86_64',
321         'arm64': 'aarch64',
322         'i386': 'i686',
323         'i486': 'i686',
324         'i686': 'i686',
325         'i786': 'i686',
326         'm68k': 'm68k',
327         'powerpc': 'powerpc',
328         'powerpc64': 'powerpc64',
329         'powerpc64le': 'powerpc64le',
330         'ppc': 'powerpc',
331         'ppc64': 'powerpc64',
332         'ppc64le': 'powerpc64le',
333         'riscv64': 'riscv64gc',
334         's390x': 's390x',
335         'x64': 'x86_64',
336         'x86': 'i686',
337         'x86-64': 'x86_64',
338         'x86_64': 'x86_64'
339     }
340
341     # Consider the direct transformation first and then the special cases
342     if cputype in cputype_mapper:
343         cputype = cputype_mapper[cputype]
344     elif cputype in {'xscale', 'arm'}:
345         cputype = 'arm'
346         if ostype == 'linux-android':
347             ostype = 'linux-androideabi'
348         elif ostype == 'unknown-freebsd':
349             cputype = subprocess.check_output(
350                 ['uname', '-p']).strip().decode(default_encoding)
351             ostype = 'unknown-freebsd'
352     elif cputype == 'armv6l':
353         cputype = 'arm'
354         if ostype == 'linux-android':
355             ostype = 'linux-androideabi'
356         else:
357             ostype += 'eabihf'
358     elif cputype in {'armv7l', 'armv8l'}:
359         cputype = 'armv7'
360         if ostype == 'linux-android':
361             ostype = 'linux-androideabi'
362         else:
363             ostype += 'eabihf'
364     elif cputype == 'mips':
365         if sys.byteorder == 'big':
366             cputype = 'mips'
367         elif sys.byteorder == 'little':
368             cputype = 'mipsel'
369         else:
370             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
371     elif cputype == 'mips64':
372         if sys.byteorder == 'big':
373             cputype = 'mips64'
374         elif sys.byteorder == 'little':
375             cputype = 'mips64el'
376         else:
377             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
378         # only the n64 ABI is supported, indicate it
379         ostype += 'abi64'
380     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
381         pass
382     else:
383         err = "unknown cpu type: {}".format(cputype)
384         sys.exit(err)
385
386     return "{}-{}".format(cputype, ostype)
387
388
389 @contextlib.contextmanager
390 def output(filepath):
391     tmp = filepath + '.tmp'
392     with open(tmp, 'w') as f:
393         yield f
394     try:
395         if os.path.exists(filepath):
396             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
397     except OSError:
398         shutil.copy2(tmp, filepath)
399         os.remove(tmp)
400         return
401     os.rename(tmp, filepath)
402
403
404 class Stage0Toolchain:
405     def __init__(self, stage0_payload):
406         self.date = stage0_payload["date"]
407         self.version = stage0_payload["version"]
408
409     def channel(self):
410         return self.version + "-" + self.date
411
412
413 class RustBuild(object):
414     """Provide all the methods required to build Rust"""
415     def __init__(self):
416         self.checksums_sha256 = {}
417         self.stage0_compiler = None
418         self.stage0_rustfmt = None
419         self._download_url = ''
420         self.build = ''
421         self.build_dir = ''
422         self.clean = False
423         self.config_toml = ''
424         self.rust_root = ''
425         self.use_locked_deps = ''
426         self.use_vendored_sources = ''
427         self.verbose = False
428         self.git_version = None
429         self.nix_deps_dir = None
430         self.rustc_commit = None
431
432     def download_toolchain(self, stage0=True, rustc_channel=None):
433         """Fetch the build system for Rust, written in Rust
434
435         This method will build a cache directory, then it will fetch the
436         tarball which has the stage0 compiler used to then bootstrap the Rust
437         compiler itself.
438
439         Each downloaded tarball is extracted, after that, the script
440         will move all the content to the right place.
441         """
442         if rustc_channel is None:
443             rustc_channel = self.stage0_compiler.version
444         bin_root = self.bin_root(stage0)
445
446         key = self.stage0_compiler.date
447         if not stage0:
448             key += str(self.rustc_commit)
449         if self.rustc(stage0).startswith(bin_root) and \
450                 (not os.path.exists(self.rustc(stage0)) or
451                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
452             if os.path.exists(bin_root):
453                 shutil.rmtree(bin_root)
454             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
455             filename = "rust-std-{}-{}{}".format(
456                 rustc_channel, self.build, tarball_suffix)
457             pattern = "rust-std-{}".format(self.build)
458             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
459             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
460                                               tarball_suffix)
461             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
462             # download-rustc doesn't need its own cargo, it can just use beta's.
463             if stage0:
464                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
465                                                 tarball_suffix)
466                 self._download_component_helper(filename, "cargo", tarball_suffix)
467                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
468             else:
469                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
470                 self._download_component_helper(
471                     filename, "rustc-dev", tarball_suffix, stage0
472                 )
473
474             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
475             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
476             lib_dir = "{}/lib".format(bin_root)
477             for lib in os.listdir(lib_dir):
478                 if lib.endswith(".so"):
479                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
480             with output(self.rustc_stamp(stage0)) as rust_stamp:
481                 rust_stamp.write(key)
482
483         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
484             not os.path.exists(self.rustfmt())
485             or self.program_out_of_date(
486                 self.rustfmt_stamp(),
487                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
488             )
489         ):
490             if self.stage0_rustfmt is not None:
491                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
492                 filename = "rustfmt-{}-{}{}".format(
493                     self.stage0_rustfmt.version, self.build, tarball_suffix,
494                 )
495                 self._download_component_helper(
496                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
497                 )
498                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
499                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
500                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
501                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
502
503     def _download_component_helper(
504         self, filename, pattern, tarball_suffix, stage0=True, key=None
505     ):
506         if key is None:
507             if stage0:
508                 key = self.stage0_compiler.date
509             else:
510                 key = self.rustc_commit
511         cache_dst = os.path.join(self.build_dir, "cache")
512         rustc_cache = os.path.join(cache_dst, key)
513         if not os.path.exists(rustc_cache):
514             os.makedirs(rustc_cache)
515
516         if stage0:
517             base = self._download_url
518             url = "dist/{}".format(key)
519         else:
520             base = "https://ci-artifacts.rust-lang.org"
521             url = "rustc-builds/{}".format(self.rustc_commit)
522         tarball = os.path.join(rustc_cache, filename)
523         if not os.path.exists(tarball):
524             get(
525                 base,
526                 "{}/{}".format(url, filename),
527                 tarball,
528                 self.checksums_sha256,
529                 verbose=self.verbose,
530                 do_verify=stage0,
531             )
532         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
533
534     def fix_bin_or_dylib(self, fname):
535         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
536         or the RPATH section, to fix the dynamic library search path
537
538         This method is only required on NixOS and uses the PatchELF utility to
539         change the interpreter/RPATH of ELF executables.
540
541         Please see https://nixos.org/patchelf.html for more information
542         """
543         default_encoding = sys.getdefaultencoding()
544         try:
545             ostype = subprocess.check_output(
546                 ['uname', '-s']).strip().decode(default_encoding)
547         except subprocess.CalledProcessError:
548             return
549         except OSError as reason:
550             if getattr(reason, 'winerror', None) is not None:
551                 return
552             raise reason
553
554         if ostype != "Linux":
555             return
556
557         # If the user has asked binaries to be patched for Nix, then
558         # don't check for NixOS or `/lib`, just continue to the patching.
559         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
560             # Use `/etc/os-release` instead of `/etc/NIXOS`.
561             # The latter one does not exist on NixOS when using tmpfs as root.
562             try:
563                 with open("/etc/os-release", "r") as f:
564                     if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
565                         return
566             except FileNotFoundError:
567                 return
568             if os.path.exists("/lib"):
569                 return
570
571         # At this point we're pretty sure the user is running NixOS or
572         # using Nix
573         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
574         print(nix_os_msg, fname)
575
576         # Only build `.nix-deps` once.
577         nix_deps_dir = self.nix_deps_dir
578         if not nix_deps_dir:
579             # Run `nix-build` to "build" each dependency (which will likely reuse
580             # the existing `/nix/store` copy, or at most download a pre-built copy).
581             #
582             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
583             # directory, but still reference the actual `/nix/store` path in the rpath
584             # as it makes it significantly more robust against changes to the location of
585             # the `.nix-deps` location.
586             #
587             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
588             # zlib: Needed as a system dependency of `libLLVM-*.so`.
589             # patchelf: Needed for patching ELF binaries (see doc comment above).
590             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
591             nix_expr = '''
592             with (import <nixpkgs> {});
593             symlinkJoin {
594               name = "rust-stage0-dependencies";
595               paths = [
596                 zlib
597                 patchelf
598                 stdenv.cc.bintools
599               ];
600             }
601             '''
602             try:
603                 subprocess.check_output([
604                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
605                 ])
606             except subprocess.CalledProcessError as reason:
607                 print("warning: failed to call nix-build:", reason)
608                 return
609             self.nix_deps_dir = nix_deps_dir
610
611         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
612         rpath_entries = [
613             # Relative default, all binary and dynamic libraries we ship
614             # appear to have this (even when `../lib` is redundant).
615             "$ORIGIN/../lib",
616             os.path.join(os.path.realpath(nix_deps_dir), "lib")
617         ]
618         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
619         if not fname.endswith(".so"):
620             # Finally, set the corret .interp for binaries
621             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
622                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
623
624         try:
625             subprocess.check_output([patchelf] + patchelf_args + [fname])
626         except subprocess.CalledProcessError as reason:
627             print("warning: failed to call patchelf:", reason)
628             return
629
630     # If `download-rustc` is set, download the most recent commit with CI artifacts
631     def maybe_download_ci_toolchain(self):
632         # If `download-rustc` is not set, default to rebuilding.
633         download_rustc = self.get_toml("download-rustc", section="rust")
634         if download_rustc is None or download_rustc == "false":
635             return None
636         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
637
638         # Handle running from a directory other than the top level
639         rev_parse = ["git", "rev-parse", "--show-toplevel"]
640         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
641         compiler = "{}/compiler/".format(top_level)
642         library = "{}/library/".format(top_level)
643
644         # Look for a version to compare to based on the current commit.
645         # Only commits merged by bors will have CI artifacts.
646         merge_base = [
647             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
648             "--first-parent", "HEAD"
649         ]
650         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
651         if not commit:
652             print("error: could not find commit hash for downloading rustc")
653             print("help: maybe your repository history is too shallow?")
654             print("help: consider disabling `download-rustc`")
655             print("help: or fetch enough history to include one upstream commit")
656             exit(1)
657
658         # Warn if there were changes to the compiler or standard library since the ancestor commit.
659         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
660         if status != 0:
661             if download_rustc == "if-unchanged":
662                 if self.verbose:
663                     print("warning: saw changes to compiler/ or library/ since {}; " \
664                           "ignoring `download-rustc`".format(commit))
665                 return None
666             print("warning: `download-rustc` is enabled, but there are changes to " \
667                   "compiler/ or library/")
668
669         if self.verbose:
670             print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
671         self.rustc_commit = commit
672         # FIXME: support downloading artifacts from the beta channel
673         self.download_toolchain(False, "nightly")
674
675     def rustc_stamp(self, stage0):
676         """Return the path for .rustc-stamp at the given stage
677
678         >>> rb = RustBuild()
679         >>> rb.build_dir = "build"
680         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
681         True
682         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
683         True
684         """
685         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
686
687     def rustfmt_stamp(self):
688         """Return the path for .rustfmt-stamp
689
690         >>> rb = RustBuild()
691         >>> rb.build_dir = "build"
692         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
693         True
694         """
695         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
696
697     def program_out_of_date(self, stamp_path, key):
698         """Check if the given program stamp is out of date"""
699         if not os.path.exists(stamp_path) or self.clean:
700             return True
701         with open(stamp_path, 'r') as stamp:
702             return key != stamp.read()
703
704     def bin_root(self, stage0):
705         """Return the binary root directory for the given stage
706
707         >>> rb = RustBuild()
708         >>> rb.build_dir = "build"
709         >>> rb.bin_root(True) == os.path.join("build", "stage0")
710         True
711         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
712         True
713
714         When the 'build' property is given should be a nested directory:
715
716         >>> rb.build = "devel"
717         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
718         True
719         """
720         if stage0:
721             subdir = "stage0"
722         else:
723             subdir = "ci-rustc"
724         return os.path.join(self.build_dir, self.build, subdir)
725
726     def get_toml(self, key, section=None):
727         """Returns the value of the given key in config.toml, otherwise returns None
728
729         >>> rb = RustBuild()
730         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
731         >>> rb.get_toml("key2")
732         'value2'
733
734         If the key does not exist, the result is None:
735
736         >>> rb.get_toml("key3") is None
737         True
738
739         Optionally also matches the section the key appears in
740
741         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
742         >>> rb.get_toml('key', 'a')
743         'value1'
744         >>> rb.get_toml('key', 'b')
745         'value2'
746         >>> rb.get_toml('key', 'c') is None
747         True
748
749         >>> rb.config_toml = 'key1 = true'
750         >>> rb.get_toml("key1")
751         'true'
752         """
753
754         cur_section = None
755         for line in self.config_toml.splitlines():
756             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
757             if section_match is not None:
758                 cur_section = section_match.group(1)
759
760             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
761             if match is not None:
762                 value = match.group(1)
763                 if section is None or section == cur_section:
764                     return self.get_string(value) or value.strip()
765         return None
766
767     def cargo(self):
768         """Return config path for cargo"""
769         return self.program_config('cargo')
770
771     def rustc(self, stage0):
772         """Return config path for rustc"""
773         return self.program_config('rustc', stage0)
774
775     def rustfmt(self):
776         """Return config path for rustfmt"""
777         if self.stage0_rustfmt is None:
778             return None
779         return self.program_config('rustfmt')
780
781     def program_config(self, program, stage0=True):
782         """Return config path for the given program at the given stage
783
784         >>> rb = RustBuild()
785         >>> rb.config_toml = 'rustc = "rustc"\\n'
786         >>> rb.program_config('rustc')
787         'rustc'
788         >>> rb.config_toml = ''
789         >>> cargo_path = rb.program_config('cargo', True)
790         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
791         ... "bin", "cargo")
792         True
793         >>> cargo_path = rb.program_config('cargo', False)
794         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
795         ... "bin", "cargo")
796         True
797         """
798         config = self.get_toml(program)
799         if config:
800             return os.path.expanduser(config)
801         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
802             program, self.exe_suffix()))
803
804     @staticmethod
805     def get_string(line):
806         """Return the value between double quotes
807
808         >>> RustBuild.get_string('    "devel"   ')
809         'devel'
810         >>> RustBuild.get_string("    'devel'   ")
811         'devel'
812         >>> RustBuild.get_string('devel') is None
813         True
814         >>> RustBuild.get_string('    "devel   ')
815         ''
816         """
817         start = line.find('"')
818         if start != -1:
819             end = start + 1 + line[start + 1:].find('"')
820             return line[start + 1:end]
821         start = line.find('\'')
822         if start != -1:
823             end = start + 1 + line[start + 1:].find('\'')
824             return line[start + 1:end]
825         return None
826
827     @staticmethod
828     def exe_suffix():
829         """Return a suffix for executables"""
830         if sys.platform == 'win32':
831             return '.exe'
832         return ''
833
834     def bootstrap_binary(self):
835         """Return the path of the bootstrap binary
836
837         >>> rb = RustBuild()
838         >>> rb.build_dir = "build"
839         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
840         ... "debug", "bootstrap")
841         True
842         """
843         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
844
845     def build_bootstrap(self):
846         """Build bootstrap"""
847         print("Building rustbuild")
848         build_dir = os.path.join(self.build_dir, "bootstrap")
849         if self.clean and os.path.exists(build_dir):
850             shutil.rmtree(build_dir)
851         env = os.environ.copy()
852         # `CARGO_BUILD_TARGET` breaks bootstrap build.
853         # See also: <https://github.com/rust-lang/rust/issues/70208>.
854         if "CARGO_BUILD_TARGET" in env:
855             del env["CARGO_BUILD_TARGET"]
856         env["CARGO_TARGET_DIR"] = build_dir
857         env["RUSTC"] = self.rustc(True)
858         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
859             (os.pathsep + env["LD_LIBRARY_PATH"]) \
860             if "LD_LIBRARY_PATH" in env else ""
861         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
862             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
863             if "DYLD_LIBRARY_PATH" in env else ""
864         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
865             (os.pathsep + env["LIBRARY_PATH"]) \
866             if "LIBRARY_PATH" in env else ""
867
868         # preserve existing RUSTFLAGS
869         env.setdefault("RUSTFLAGS", "")
870         build_section = "target.{}".format(self.build)
871         target_features = []
872         if self.get_toml("crt-static", build_section) == "true":
873             target_features += ["+crt-static"]
874         elif self.get_toml("crt-static", build_section) == "false":
875             target_features += ["-crt-static"]
876         if target_features:
877             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
878         target_linker = self.get_toml("linker", build_section)
879         if target_linker is not None:
880             env["RUSTFLAGS"] += " -C linker=" + target_linker
881         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
882         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
883         if self.get_toml("deny-warnings", "rust") != "false":
884             env["RUSTFLAGS"] += " -Dwarnings"
885
886         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
887             os.pathsep + env["PATH"]
888         if not os.path.isfile(self.cargo()):
889             raise Exception("no cargo executable found at `{}`".format(
890                 self.cargo()))
891         args = [self.cargo(), "build", "--manifest-path",
892                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
893         for _ in range(0, self.verbose):
894             args.append("--verbose")
895         if self.use_locked_deps:
896             args.append("--locked")
897         if self.use_vendored_sources:
898             args.append("--frozen")
899         run(args, env=env, verbose=self.verbose)
900
901     def build_triple(self):
902         """Build triple as in LLVM
903
904         Note that `default_build_triple` is moderately expensive,
905         so use `self.build` where possible.
906         """
907         config = self.get_toml('build')
908         if config:
909             return config
910         return default_build_triple(self.verbose)
911
912     def check_submodule(self, module, slow_submodules):
913         if not slow_submodules:
914             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
915                                            cwd=os.path.join(self.rust_root, module),
916                                            stdout=subprocess.PIPE)
917             return checked_out
918         else:
919             return None
920
921     def update_submodule(self, module, checked_out, recorded_submodules):
922         module_path = os.path.join(self.rust_root, module)
923
924         if checked_out is not None:
925             default_encoding = sys.getdefaultencoding()
926             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
927             if recorded_submodules[module] == checked_out:
928                 return
929
930         print("Updating submodule", module)
931
932         run(["git", "submodule", "-q", "sync", module],
933             cwd=self.rust_root, verbose=self.verbose)
934
935         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
936         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
937             update_args.append("--progress")
938         update_args.append(module)
939         try:
940             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
941         except RuntimeError:
942             print("Failed updating submodule. This is probably due to uncommitted local changes.")
943             print('Either stash the changes by running "git stash" within the submodule\'s')
944             print('directory, reset them by running "git reset --hard", or commit them.')
945             print("To reset all submodules' changes run", end=" ")
946             print('"git submodule foreach --recursive git reset --hard".')
947             raise SystemExit(1)
948
949         run(["git", "reset", "-q", "--hard"],
950             cwd=module_path, verbose=self.verbose)
951         run(["git", "clean", "-qdfx"],
952             cwd=module_path, verbose=self.verbose)
953
954     def update_submodules(self):
955         """Update submodules"""
956         has_git = os.path.exists(os.path.join(self.rust_root, ".git"))
957         # This just arbitrarily checks for cargo, but any workspace member in
958         # a submodule would work.
959         has_submodules = os.path.exists(os.path.join(self.rust_root, "src/tools/cargo/Cargo.toml"))
960         if not has_git and not has_submodules:
961             print("This is not a git repository, and the requisite git submodules were not found.")
962             print("If you downloaded the source from https://github.com/rust-lang/rust/releases,")
963             print("those sources will not work. Instead, consider downloading from the source")
964             print("releases linked at")
965             print("https://forge.rust-lang.org/infra/other-installation-methods.html#source-code")
966             print("or clone the repository at https://github.com/rust-lang/rust/.")
967             raise SystemExit(1)
968         if not has_git or self.get_toml('submodules') == "false":
969             return
970
971         default_encoding = sys.getdefaultencoding()
972
973         # check the existence and version of 'git' command
974         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
975         self.git_version = distutils.version.LooseVersion(git_version_str)
976
977         slow_submodules = self.get_toml('fast-submodules') == "false"
978         start_time = time()
979         if slow_submodules:
980             print('Unconditionally updating submodules')
981         else:
982             print('Updating only changed submodules')
983         default_encoding = sys.getdefaultencoding()
984         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
985         # currently requires everything in a workspace to be "locally present" when starting a
986         # build, and will give a hard error if any Cargo.toml files are missing.
987         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
988         #   share a workspace with any tools - maybe it could be excluded from the workspace?
989         #   That will still require cloning the submodules the second you check the standard
990         #   library, though...
991         # FIXME: Is there a way to avoid hard-coding the submodules required?
992         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
993         submodules = [
994             "src/tools/rust-installer",
995             "src/tools/cargo",
996             "src/tools/rls",
997             "src/tools/miri",
998             "library/backtrace",
999             "library/stdarch"
1000         ]
1001         # If build.vendor is set in config.toml, we must update rust-analyzer also.
1002         # Otherwise, the bootstrap will fail (#96456).
1003         if self.use_vendored_sources:
1004             submodules.append("src/tools/rust-analyzer")
1005         filtered_submodules = []
1006         submodules_names = []
1007         for module in submodules:
1008             check = self.check_submodule(module, slow_submodules)
1009             filtered_submodules.append((module, check))
1010             submodules_names.append(module)
1011         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1012                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1013         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1014         # { filename: hash }
1015         recorded_submodules = {}
1016         for data in recorded:
1017             # [mode, kind, hash, filename]
1018             data = data.split()
1019             recorded_submodules[data[3]] = data[2]
1020         for module in filtered_submodules:
1021             self.update_submodule(module[0], module[1], recorded_submodules)
1022         print("  Submodules updated in %.2f seconds" % (time() - start_time))
1023
1024     def set_dist_environment(self, url):
1025         """Set download URL for normal environment"""
1026         if 'RUSTUP_DIST_SERVER' in os.environ:
1027             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1028         else:
1029             self._download_url = url
1030
1031     def check_vendored_status(self):
1032         """Check that vendoring is configured properly"""
1033         vendor_dir = os.path.join(self.rust_root, 'vendor')
1034         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1035             if os.getuid() == 0:
1036                 self.use_vendored_sources = True
1037                 print('info: looks like you\'re trying to run this command as root')
1038                 print('      and so in order to preserve your $HOME this will now')
1039                 print('      use vendored sources by default.')
1040                 if not os.path.exists(vendor_dir):
1041                     print('error: vendoring required, but vendor directory does not exist.')
1042                     print('       Run `cargo vendor` without sudo to initialize the '
1043                           'vendor directory.')
1044                     raise Exception("{} not found".format(vendor_dir))
1045
1046         if self.use_vendored_sources:
1047             config = ("[source.crates-io]\n"
1048                       "replace-with = 'vendored-sources'\n"
1049                       "registry = 'https://example.com'\n"
1050                       "\n"
1051                       "[source.vendored-sources]\n"
1052                       "directory = '{}/vendor'\n"
1053                       .format(self.rust_root))
1054             if not os.path.exists('.cargo'):
1055                 os.makedirs('.cargo')
1056                 with output('.cargo/config') as cargo_config:
1057                     cargo_config.write(config)
1058             else:
1059                 print('info: using vendored source, but .cargo/config is already present.')
1060                 print('      Reusing the current configuration file. But you may want to '
1061                       'configure vendoring like this:')
1062                 print(config)
1063         else:
1064             if os.path.exists('.cargo'):
1065                 shutil.rmtree('.cargo')
1066
1067     def ensure_vendored(self):
1068         """Ensure that the vendored sources are available if needed"""
1069         vendor_dir = os.path.join(self.rust_root, 'vendor')
1070         # Note that this does not handle updating the vendored dependencies if
1071         # the rust git repository is updated. Normal development usually does
1072         # not use vendoring, so hopefully this isn't too much of a problem.
1073         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1074             run([
1075                 self.cargo(),
1076                 "vendor",
1077                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1078                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1079             ], verbose=self.verbose, cwd=self.rust_root)
1080
1081
1082 def bootstrap(help_triggered):
1083     """Configure, fetch, build and run the initial bootstrap"""
1084
1085     # If the user is asking for help, let them know that the whole download-and-build
1086     # process has to happen before anything is printed out.
1087     if help_triggered:
1088         print("info: Downloading and building bootstrap before processing --help")
1089         print("      command. See src/bootstrap/README.md for help with common")
1090         print("      commands.")
1091
1092     parser = argparse.ArgumentParser(description='Build rust')
1093     parser.add_argument('--config')
1094     parser.add_argument('--build')
1095     parser.add_argument('--clean', action='store_true')
1096     parser.add_argument('-v', '--verbose', action='count', default=0)
1097
1098     args = [a for a in sys.argv if a != '-h' and a != '--help']
1099     args, _ = parser.parse_known_args(args)
1100
1101     # Configure initial bootstrap
1102     build = RustBuild()
1103     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1104     build.verbose = args.verbose
1105     build.clean = args.clean
1106
1107     # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1108     # then `config.toml` in the root directory.
1109     toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1110     using_default_path = toml_path is None
1111     if using_default_path:
1112         toml_path = 'config.toml'
1113         if not os.path.exists(toml_path):
1114             toml_path = os.path.join(build.rust_root, toml_path)
1115
1116     # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1117     # but not if `config.toml` hasn't been created.
1118     if not using_default_path or os.path.exists(toml_path):
1119         with open(toml_path) as config:
1120             build.config_toml = config.read()
1121
1122     profile = build.get_toml('profile')
1123     if profile is not None:
1124         include_file = 'config.{}.toml'.format(profile)
1125         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1126         include_path = os.path.join(include_dir, include_file)
1127         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1128         # specific key, so appending our defaults at the end allows the user to override them
1129         with open(include_path) as included_toml:
1130             build.config_toml += os.linesep + included_toml.read()
1131
1132     config_verbose = build.get_toml('verbose', 'build')
1133     if config_verbose is not None:
1134         build.verbose = max(build.verbose, int(config_verbose))
1135
1136     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1137
1138     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1139
1140     build.check_vendored_status()
1141
1142     build_dir = build.get_toml('build-dir', 'build') or 'build'
1143     build.build_dir = os.path.abspath(build_dir)
1144
1145     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1146         data = json.load(f)
1147     build.checksums_sha256 = data["checksums_sha256"]
1148     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1149     if data.get("rustfmt") is not None:
1150         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1151
1152     build.set_dist_environment(data["dist_server"])
1153
1154     build.build = args.build or build.build_triple()
1155
1156     # Acquire the lock before doing any build actions
1157     # The lock is released when `lock` is dropped
1158     if not os.path.exists(build.build_dir):
1159         os.makedirs(build.build_dir)
1160     lock = acquire_lock(build.build_dir)
1161     build.update_submodules()
1162
1163     # Fetch/build the bootstrap
1164     build.download_toolchain()
1165     # Download the master compiler if `download-rustc` is set
1166     build.maybe_download_ci_toolchain()
1167     sys.stdout.flush()
1168     build.ensure_vendored()
1169     build.build_bootstrap()
1170     sys.stdout.flush()
1171
1172     # Run the bootstrap
1173     args = [build.bootstrap_binary()]
1174     args.extend(sys.argv[1:])
1175     env = os.environ.copy()
1176     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1177     env["BOOTSTRAP_PYTHON"] = sys.executable
1178     env["RUSTC_BOOTSTRAP"] = '1'
1179     if build.rustc_commit is not None:
1180         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1181     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1182
1183
1184 def main():
1185     """Entry point for the bootstrap process"""
1186     start_time = time()
1187
1188     # x.py help <cmd> ...
1189     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1190         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1191
1192     help_triggered = (
1193         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1194     try:
1195         bootstrap(help_triggered)
1196         if not help_triggered:
1197             print("Build completed successfully in {}".format(
1198                 format_build_time(time() - start_time)))
1199     except (SystemExit, KeyboardInterrupt) as error:
1200         if hasattr(error, 'code') and isinstance(error.code, int):
1201             exit_code = error.code
1202         else:
1203             exit_code = 1
1204             print(error)
1205         if not help_triggered:
1206             print("Build completed unsuccessfully in {}".format(
1207                 format_build_time(time() - start_time)))
1208         sys.exit(exit_code)
1209
1210
1211 if __name__ == '__main__':
1212     main()