]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
Rollup merge of #84466 - jyn514:prim-str, r=GuillaumeGomez
[rust.git] / src / bootstrap / bootstrap.py
1 from __future__ import absolute_import, division, print_function
2 import argparse
3 import contextlib
4 import datetime
5 import distutils.version
6 import hashlib
7 import os
8 import re
9 import shutil
10 import subprocess
11 import sys
12 import tarfile
13 import tempfile
14
15 from time import time
16
17 def support_xz():
18     try:
19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20             temp_path = temp_file.name
21         with tarfile.open(temp_path, "w:xz"):
22             pass
23         return True
24     except tarfile.CompressionError:
25         return False
26
27 def get(url, path, verbose=False, do_verify=True):
28     suffix = '.sha256'
29     sha_url = url + suffix
30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31         temp_path = temp_file.name
32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33         sha_path = sha_file.name
34
35     try:
36         if do_verify:
37             download(sha_path, sha_url, False, verbose)
38             if os.path.exists(path):
39                 if verify(path, sha_path, False):
40                     if verbose:
41                         print("using already-download file", path)
42                     return
43                 else:
44                     if verbose:
45                         print("ignoring already-download file",
46                             path, "due to failed verification")
47                     os.unlink(path)
48         download(temp_path, url, True, verbose)
49         if do_verify and not verify(temp_path, sha_path, verbose):
50             raise RuntimeError("failed verification")
51         if verbose:
52             print("moving {} to {}".format(temp_path, path))
53         shutil.move(temp_path, path)
54     finally:
55         delete_if_present(sha_path, verbose)
56         delete_if_present(temp_path, verbose)
57
58
59 def delete_if_present(path, verbose):
60     """Remove the given file if present"""
61     if os.path.isfile(path):
62         if verbose:
63             print("removing", path)
64         os.unlink(path)
65
66
67 def download(path, url, probably_big, verbose):
68     for _ in range(0, 4):
69         try:
70             _download(path, url, probably_big, verbose, True)
71             return
72         except RuntimeError:
73             print("\nspurious failure, trying again")
74     _download(path, url, probably_big, verbose, False)
75
76
77 def _download(path, url, probably_big, verbose, exception):
78     if probably_big or verbose:
79         print("downloading {}".format(url))
80     # see http://serverfault.com/questions/301128/how-to-download
81     if sys.platform == 'win32':
82         run(["PowerShell.exe", "/nologo", "-Command",
83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
85             verbose=verbose,
86             exception=exception)
87     else:
88         if probably_big or verbose:
89             option = "-#"
90         else:
91             option = "-s"
92         require(["curl", "--version"])
93         run(["curl", option,
94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
96              "--retry", "3", "-Sf", "-o", path, url],
97             verbose=verbose,
98             exception=exception)
99
100
101 def verify(path, sha_path, verbose):
102     """Check if the sha256 sum of the given path is valid"""
103     if verbose:
104         print("verifying", path)
105     with open(path, "rb") as source:
106         found = hashlib.sha256(source.read()).hexdigest()
107     with open(sha_path, "r") as sha256sum:
108         expected = sha256sum.readline().split()[0]
109     verified = found == expected
110     if not verified:
111         print("invalid checksum:\n"
112               "    found:    {}\n"
113               "    expected: {}".format(found, expected))
114     return verified
115
116
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118     """Unpack the given tarball file"""
119     print("extracting", tarball)
120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
121     with contextlib.closing(tarfile.open(tarball)) as tar:
122         for member in tar.getnames():
123             if "/" not in member:
124                 continue
125             name = member.replace(fname + "/", "", 1)
126             if match is not None and not name.startswith(match):
127                 continue
128             name = name[len(match) + 1:]
129
130             dst_path = os.path.join(dst, name)
131             if verbose:
132                 print("  extracting", member)
133             tar.extract(member, dst)
134             src_path = os.path.join(dst, member)
135             if os.path.isdir(src_path) and os.path.exists(dst_path):
136                 continue
137             shutil.move(src_path, dst_path)
138     shutil.rmtree(os.path.join(dst, fname))
139
140
141 def run(args, verbose=False, exception=False, **kwargs):
142     """Run a child program in a new process"""
143     if verbose:
144         print("running: " + ' '.join(args))
145     sys.stdout.flush()
146     # Use Popen here instead of call() as it apparently allows powershell on
147     # Windows to not lock up waiting for input presumably.
148     ret = subprocess.Popen(args, **kwargs)
149     code = ret.wait()
150     if code != 0:
151         err = "failed to run: " + ' '.join(args)
152         if verbose or exception:
153             raise RuntimeError(err)
154         sys.exit(err)
155
156
157 def require(cmd, exit=True):
158     '''Run a command, returning its output.
159     On error,
160         If `exit` is `True`, exit the process.
161         Otherwise, return None.'''
162     try:
163         return subprocess.check_output(cmd).strip()
164     except (subprocess.CalledProcessError, OSError) as exc:
165         if not exit:
166             return None
167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168         print("Please make sure it's installed and in the path.")
169         sys.exit(1)
170
171
172 def stage0_data(rust_root):
173     """Build a dictionary from stage0.txt"""
174     nightlies = os.path.join(rust_root, "src/stage0.txt")
175     with open(nightlies, 'r') as nightlies:
176         lines = [line.rstrip() for line in nightlies
177                  if not line.startswith("#")]
178         return dict([line.split(": ", 1) for line in lines if line])
179
180
181 def format_build_time(duration):
182     """Return a nicer format for build time
183
184     >>> format_build_time('300')
185     '0:05:00'
186     """
187     return str(datetime.timedelta(seconds=int(duration)))
188
189
190 def default_build_triple(verbose):
191     """Build triple as in LLVM"""
192     # If the user already has a host build triple with an existing `rustc`
193     # install, use their preference. This fixes most issues with Windows builds
194     # being detected as GNU instead of MSVC.
195     default_encoding = sys.getdefaultencoding()
196     try:
197         version = subprocess.check_output(["rustc", "--version", "--verbose"],
198                 stderr=subprocess.DEVNULL)
199         version = version.decode(default_encoding)
200         host = next(x for x in version.split('\n') if x.startswith("host: "))
201         triple = host.split("host: ")[1]
202         if verbose:
203             print("detected default triple {}".format(triple))
204         return triple
205     except Exception as e:
206         if verbose:
207             print("rustup not detected: {}".format(e))
208             print("falling back to auto-detect")
209
210     required = sys.platform != 'win32'
211     ostype = require(["uname", "-s"], exit=required)
212     cputype = require(['uname', '-m'], exit=required)
213
214     # If we do not have `uname`, assume Windows.
215     if ostype is None or cputype is None:
216         return 'x86_64-pc-windows-msvc'
217
218     ostype = ostype.decode(default_encoding)
219     cputype = cputype.decode(default_encoding)
220
221     # The goal here is to come up with the same triple as LLVM would,
222     # at least for the subset of platforms we're willing to target.
223     ostype_mapper = {
224         'Darwin': 'apple-darwin',
225         'DragonFly': 'unknown-dragonfly',
226         'FreeBSD': 'unknown-freebsd',
227         'Haiku': 'unknown-haiku',
228         'NetBSD': 'unknown-netbsd',
229         'OpenBSD': 'unknown-openbsd'
230     }
231
232     # Consider the direct transformation first and then the special cases
233     if ostype in ostype_mapper:
234         ostype = ostype_mapper[ostype]
235     elif ostype == 'Linux':
236         os_from_sp = subprocess.check_output(
237             ['uname', '-o']).strip().decode(default_encoding)
238         if os_from_sp == 'Android':
239             ostype = 'linux-android'
240         else:
241             ostype = 'unknown-linux-gnu'
242     elif ostype == 'SunOS':
243         ostype = 'pc-solaris'
244         # On Solaris, uname -m will return a machine classification instead
245         # of a cpu type, so uname -p is recommended instead.  However, the
246         # output from that option is too generic for our purposes (it will
247         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
248         # must be used instead.
249         cputype = require(['isainfo', '-k']).decode(default_encoding)
250         # sparc cpus have sun as a target vendor
251         if 'sparc' in cputype:
252             ostype = 'sun-solaris'
253     elif ostype.startswith('MINGW'):
254         # msys' `uname` does not print gcc configuration, but prints msys
255         # configuration. so we cannot believe `uname -m`:
256         # msys1 is always i686 and msys2 is always x86_64.
257         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
258         # MINGW64 on x86_64.
259         ostype = 'pc-windows-gnu'
260         cputype = 'i686'
261         if os.environ.get('MSYSTEM') == 'MINGW64':
262             cputype = 'x86_64'
263     elif ostype.startswith('MSYS'):
264         ostype = 'pc-windows-gnu'
265     elif ostype.startswith('CYGWIN_NT'):
266         cputype = 'i686'
267         if ostype.endswith('WOW64'):
268             cputype = 'x86_64'
269         ostype = 'pc-windows-gnu'
270     elif sys.platform == 'win32':
271         # Some Windows platforms might have a `uname` command that returns a
272         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
273         # these cases, fall back to using sys.platform.
274         return 'x86_64-pc-windows-msvc'
275     else:
276         err = "unknown OS type: {}".format(ostype)
277         sys.exit(err)
278
279     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
280         cputype = subprocess.check_output(
281               ['uname', '-p']).strip().decode(default_encoding)
282     cputype_mapper = {
283         'BePC': 'i686',
284         'aarch64': 'aarch64',
285         'amd64': 'x86_64',
286         'arm64': 'aarch64',
287         'i386': 'i686',
288         'i486': 'i686',
289         'i686': 'i686',
290         'i786': 'i686',
291         'powerpc': 'powerpc',
292         'powerpc64': 'powerpc64',
293         'powerpc64le': 'powerpc64le',
294         'ppc': 'powerpc',
295         'ppc64': 'powerpc64',
296         'ppc64le': 'powerpc64le',
297         's390x': 's390x',
298         'x64': 'x86_64',
299         'x86': 'i686',
300         'x86-64': 'x86_64',
301         'x86_64': 'x86_64'
302     }
303
304     # Consider the direct transformation first and then the special cases
305     if cputype in cputype_mapper:
306         cputype = cputype_mapper[cputype]
307     elif cputype in {'xscale', 'arm'}:
308         cputype = 'arm'
309         if ostype == 'linux-android':
310             ostype = 'linux-androideabi'
311         elif ostype == 'unknown-freebsd':
312             cputype = subprocess.check_output(
313                 ['uname', '-p']).strip().decode(default_encoding)
314             ostype = 'unknown-freebsd'
315     elif cputype == 'armv6l':
316         cputype = 'arm'
317         if ostype == 'linux-android':
318             ostype = 'linux-androideabi'
319         else:
320             ostype += 'eabihf'
321     elif cputype in {'armv7l', 'armv8l'}:
322         cputype = 'armv7'
323         if ostype == 'linux-android':
324             ostype = 'linux-androideabi'
325         else:
326             ostype += 'eabihf'
327     elif cputype == 'mips':
328         if sys.byteorder == 'big':
329             cputype = 'mips'
330         elif sys.byteorder == 'little':
331             cputype = 'mipsel'
332         else:
333             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
334     elif cputype == 'mips64':
335         if sys.byteorder == 'big':
336             cputype = 'mips64'
337         elif sys.byteorder == 'little':
338             cputype = 'mips64el'
339         else:
340             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
341         # only the n64 ABI is supported, indicate it
342         ostype += 'abi64'
343     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
344         pass
345     else:
346         err = "unknown cpu type: {}".format(cputype)
347         sys.exit(err)
348
349     return "{}-{}".format(cputype, ostype)
350
351
352 @contextlib.contextmanager
353 def output(filepath):
354     tmp = filepath + '.tmp'
355     with open(tmp, 'w') as f:
356         yield f
357     try:
358         if os.path.exists(filepath):
359             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
360     except OSError:
361         shutil.copy2(tmp, filepath)
362         os.remove(tmp)
363         return
364     os.rename(tmp, filepath)
365
366
367 class RustBuild(object):
368     """Provide all the methods required to build Rust"""
369     def __init__(self):
370         self.date = ''
371         self._download_url = ''
372         self.rustc_channel = ''
373         self.rustfmt_channel = ''
374         self.build = ''
375         self.build_dir = ''
376         self.clean = False
377         self.config_toml = ''
378         self.rust_root = ''
379         self.use_locked_deps = ''
380         self.use_vendored_sources = ''
381         self.verbose = False
382         self.git_version = None
383         self.nix_deps_dir = None
384         self.rustc_commit = None
385
386     def download_toolchain(self, stage0=True, rustc_channel=None):
387         """Fetch the build system for Rust, written in Rust
388
389         This method will build a cache directory, then it will fetch the
390         tarball which has the stage0 compiler used to then bootstrap the Rust
391         compiler itself.
392
393         Each downloaded tarball is extracted, after that, the script
394         will move all the content to the right place.
395         """
396         if rustc_channel is None:
397             rustc_channel = self.rustc_channel
398         rustfmt_channel = self.rustfmt_channel
399         bin_root = self.bin_root(stage0)
400
401         key = self.date
402         if not stage0:
403             key += str(self.rustc_commit)
404         if self.rustc(stage0).startswith(bin_root) and \
405                 (not os.path.exists(self.rustc(stage0)) or
406                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
407             if os.path.exists(bin_root):
408                 shutil.rmtree(bin_root)
409             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
410             filename = "rust-std-{}-{}{}".format(
411                 rustc_channel, self.build, tarball_suffix)
412             pattern = "rust-std-{}".format(self.build)
413             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
414             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
415                                               tarball_suffix)
416             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
417             # download-rustc doesn't need its own cargo, it can just use beta's.
418             if stage0:
419                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
420                                                 tarball_suffix)
421                 self._download_component_helper(filename, "cargo", tarball_suffix)
422                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
423             else:
424                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
425                 self._download_component_helper(
426                     filename, "rustc-dev", tarball_suffix, stage0
427                 )
428
429             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
430             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
431             lib_dir = "{}/lib".format(bin_root)
432             for lib in os.listdir(lib_dir):
433                 if lib.endswith(".so"):
434                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
435             with output(self.rustc_stamp(stage0)) as rust_stamp:
436                 rust_stamp.write(key)
437
438         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
439             not os.path.exists(self.rustfmt())
440             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
441         ):
442             if rustfmt_channel:
443                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
444                 [channel, date] = rustfmt_channel.split('-', 1)
445                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
446                 self._download_component_helper(
447                     filename, "rustfmt-preview", tarball_suffix, key=date
448                 )
449                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
450                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
451                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
452                     rustfmt_stamp.write(self.rustfmt_channel)
453
454         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
455         if self.downloading_llvm() and stage0:
456             # We want the most recent LLVM submodule update to avoid downloading
457             # LLVM more often than necessary.
458             #
459             # This git command finds that commit SHA, looking for bors-authored
460             # merges that modified src/llvm-project.
461             #
462             # This works even in a repository that has not yet initialized
463             # submodules.
464             top_level = subprocess.check_output([
465                 "git", "rev-parse", "--show-toplevel",
466             ]).decode(sys.getdefaultencoding()).strip()
467             llvm_sha = subprocess.check_output([
468                 "git", "log", "--author=bors", "--format=%H", "-n1",
469                 "-m", "--first-parent",
470                 "--",
471                 "{}/src/llvm-project".format(top_level),
472                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
473                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
474                 "{}/src/version".format(top_level)
475             ]).decode(sys.getdefaultencoding()).strip()
476             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
477             llvm_root = self.llvm_root()
478             llvm_lib = os.path.join(llvm_root, "lib")
479             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
480                 self._download_ci_llvm(llvm_sha, llvm_assertions)
481                 for binary in ["llvm-config", "FileCheck"]:
482                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
483                 for lib in os.listdir(llvm_lib):
484                     if lib.endswith(".so"):
485                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
486                 with output(self.llvm_stamp()) as llvm_stamp:
487                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
488
489     def downloading_llvm(self):
490         opt = self.get_toml('download-ci-llvm', 'llvm')
491         # This is currently all tier 1 targets (since others may not have CI
492         # artifacts)
493         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
494         supported_platforms = [
495             "aarch64-unknown-linux-gnu",
496             "i686-pc-windows-gnu",
497             "i686-pc-windows-msvc",
498             "i686-unknown-linux-gnu",
499             "x86_64-unknown-linux-gnu",
500             "x86_64-apple-darwin",
501             "x86_64-pc-windows-gnu",
502             "x86_64-pc-windows-msvc",
503         ]
504         return opt == "true" \
505             or (opt == "if-available" and self.build in supported_platforms)
506
507     def _download_component_helper(
508         self, filename, pattern, tarball_suffix, stage0=True, key=None
509     ):
510         if key is None:
511             if stage0:
512                 key = self.date
513             else:
514                 key = self.rustc_commit
515         cache_dst = os.path.join(self.build_dir, "cache")
516         rustc_cache = os.path.join(cache_dst, key)
517         if not os.path.exists(rustc_cache):
518             os.makedirs(rustc_cache)
519
520         if stage0:
521             url = "{}/dist/{}".format(self._download_url, key)
522         else:
523             url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
524         tarball = os.path.join(rustc_cache, filename)
525         if not os.path.exists(tarball):
526             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0)
527         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
528
529     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
530         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
531         cache_dst = os.path.join(self.build_dir, "cache")
532         rustc_cache = os.path.join(cache_dst, cache_prefix)
533         if not os.path.exists(rustc_cache):
534             os.makedirs(rustc_cache)
535
536         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
537         if llvm_assertions:
538             url = url.replace('rustc-builds', 'rustc-builds-alt')
539         # ci-artifacts are only stored as .xz, not .gz
540         if not support_xz():
541             print("error: XZ support is required to download LLVM")
542             print("help: consider disabling `download-ci-llvm` or using python3")
543             exit(1)
544         tarball_suffix = '.tar.xz'
545         filename = "rust-dev-nightly-" + self.build + tarball_suffix
546         tarball = os.path.join(rustc_cache, filename)
547         if not os.path.exists(tarball):
548             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
549         unpack(tarball, tarball_suffix, self.llvm_root(),
550                 match="rust-dev",
551                 verbose=self.verbose)
552
553     def fix_bin_or_dylib(self, fname):
554         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
555         or the RPATH section, to fix the dynamic library search path
556
557         This method is only required on NixOS and uses the PatchELF utility to
558         change the interpreter/RPATH of ELF executables.
559
560         Please see https://nixos.org/patchelf.html for more information
561         """
562         default_encoding = sys.getdefaultencoding()
563         try:
564             ostype = subprocess.check_output(
565                 ['uname', '-s']).strip().decode(default_encoding)
566         except subprocess.CalledProcessError:
567             return
568         except OSError as reason:
569             if getattr(reason, 'winerror', None) is not None:
570                 return
571             raise reason
572
573         if ostype != "Linux":
574             return
575
576         if not os.path.exists("/etc/NIXOS"):
577             return
578         if os.path.exists("/lib"):
579             return
580
581         # At this point we're pretty sure the user is running NixOS
582         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
583         print(nix_os_msg, fname)
584
585         # Only build `.nix-deps` once.
586         nix_deps_dir = self.nix_deps_dir
587         if not nix_deps_dir:
588             # Run `nix-build` to "build" each dependency (which will likely reuse
589             # the existing `/nix/store` copy, or at most download a pre-built copy).
590             #
591             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
592             # directory, but still reference the actual `/nix/store` path in the rpath
593             # as it makes it significantly more robust against changes to the location of
594             # the `.nix-deps` location.
595             #
596             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
597             # zlib: Needed as a system dependency of `libLLVM-*.so`.
598             # patchelf: Needed for patching ELF binaries (see doc comment above).
599             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
600             nix_expr = '''
601             with (import <nixpkgs> {});
602             symlinkJoin {
603               name = "rust-stage0-dependencies";
604               paths = [
605                 zlib
606                 patchelf
607                 stdenv.cc.bintools
608               ];
609             }
610             '''
611             try:
612                 subprocess.check_output([
613                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
614                 ])
615             except subprocess.CalledProcessError as reason:
616                 print("warning: failed to call nix-build:", reason)
617                 return
618             self.nix_deps_dir = nix_deps_dir
619
620         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
621         rpath_entries = [
622             # Relative default, all binary and dynamic libraries we ship
623             # appear to have this (even when `../lib` is redundant).
624             "$ORIGIN/../lib",
625             os.path.join(os.path.realpath(nix_deps_dir), "lib")
626         ]
627         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
628         if not fname.endswith(".so"):
629             # Finally, set the corret .interp for binaries
630             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
631                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
632
633         try:
634             subprocess.check_output([patchelf] + patchelf_args + [fname])
635         except subprocess.CalledProcessError as reason:
636             print("warning: failed to call patchelf:", reason)
637             return
638
639     # If `download-rustc` is set, download the most recent commit with CI artifacts
640     def maybe_download_ci_toolchain(self):
641         # If `download-rustc` is not set, default to rebuilding.
642         download_rustc = self.get_toml("download-rustc", section="rust")
643         if download_rustc is None or download_rustc == "false":
644             return None
645         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
646
647         # Handle running from a directory other than the top level
648         rev_parse = ["git", "rev-parse", "--show-toplevel"]
649         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
650         compiler = "{}/compiler/".format(top_level)
651
652         # Look for a version to compare to based on the current commit.
653         # Only commits merged by bors will have CI artifacts.
654         merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"]
655         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
656
657         # Warn if there were changes to the compiler since the ancestor commit.
658         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
659         if status != 0:
660             if download_rustc == "if-unchanged":
661                 return None
662             print("warning: `download-rustc` is enabled, but there are changes to compiler/")
663
664         if self.verbose:
665             print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
666         self.rustc_commit = commit
667         # FIXME: support downloading artifacts from the beta channel
668         self.download_toolchain(False, "nightly")
669
670     def rustc_stamp(self, stage0):
671         """Return the path for .rustc-stamp at the given stage
672
673         >>> rb = RustBuild()
674         >>> rb.build_dir = "build"
675         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
676         True
677         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
678         True
679         """
680         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
681
682     def rustfmt_stamp(self):
683         """Return the path for .rustfmt-stamp
684
685         >>> rb = RustBuild()
686         >>> rb.build_dir = "build"
687         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
688         True
689         """
690         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
691
692     def llvm_stamp(self):
693         """Return the path for .rustfmt-stamp
694
695         >>> rb = RustBuild()
696         >>> rb.build_dir = "build"
697         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
698         True
699         """
700         return os.path.join(self.llvm_root(), '.llvm-stamp')
701
702
703     def program_out_of_date(self, stamp_path, key):
704         """Check if the given program stamp is out of date"""
705         if not os.path.exists(stamp_path) or self.clean:
706             return True
707         with open(stamp_path, 'r') as stamp:
708             return key != stamp.read()
709
710     def bin_root(self, stage0):
711         """Return the binary root directory for the given stage
712
713         >>> rb = RustBuild()
714         >>> rb.build_dir = "build"
715         >>> rb.bin_root(True) == os.path.join("build", "stage0")
716         True
717         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
718         True
719
720         When the 'build' property is given should be a nested directory:
721
722         >>> rb.build = "devel"
723         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
724         True
725         """
726         if stage0:
727             subdir = "stage0"
728         else:
729             subdir = "ci-rustc"
730         return os.path.join(self.build_dir, self.build, subdir)
731
732     def llvm_root(self):
733         """Return the CI LLVM root directory
734
735         >>> rb = RustBuild()
736         >>> rb.build_dir = "build"
737         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
738         True
739
740         When the 'build' property is given should be a nested directory:
741
742         >>> rb.build = "devel"
743         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
744         True
745         """
746         return os.path.join(self.build_dir, self.build, "ci-llvm")
747
748     def get_toml(self, key, section=None):
749         """Returns the value of the given key in config.toml, otherwise returns None
750
751         >>> rb = RustBuild()
752         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
753         >>> rb.get_toml("key2")
754         'value2'
755
756         If the key does not exists, the result is None:
757
758         >>> rb.get_toml("key3") is None
759         True
760
761         Optionally also matches the section the key appears in
762
763         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
764         >>> rb.get_toml('key', 'a')
765         'value1'
766         >>> rb.get_toml('key', 'b')
767         'value2'
768         >>> rb.get_toml('key', 'c') is None
769         True
770
771         >>> rb.config_toml = 'key1 = true'
772         >>> rb.get_toml("key1")
773         'true'
774         """
775
776         cur_section = None
777         for line in self.config_toml.splitlines():
778             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
779             if section_match is not None:
780                 cur_section = section_match.group(1)
781
782             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
783             if match is not None:
784                 value = match.group(1)
785                 if section is None or section == cur_section:
786                     return self.get_string(value) or value.strip()
787         return None
788
789     def cargo(self):
790         """Return config path for cargo"""
791         return self.program_config('cargo')
792
793     def rustc(self, stage0):
794         """Return config path for rustc"""
795         return self.program_config('rustc', stage0)
796
797     def rustfmt(self):
798         """Return config path for rustfmt"""
799         if not self.rustfmt_channel:
800             return None
801         return self.program_config('rustfmt')
802
803     def program_config(self, program, stage0=True):
804         """Return config path for the given program at the given stage
805
806         >>> rb = RustBuild()
807         >>> rb.config_toml = 'rustc = "rustc"\\n'
808         >>> rb.program_config('rustc')
809         'rustc'
810         >>> rb.config_toml = ''
811         >>> cargo_path = rb.program_config('cargo', True)
812         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
813         ... "bin", "cargo")
814         True
815         >>> cargo_path = rb.program_config('cargo', False)
816         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
817         ... "bin", "cargo")
818         True
819         """
820         config = self.get_toml(program)
821         if config:
822             return os.path.expanduser(config)
823         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
824             program, self.exe_suffix()))
825
826     @staticmethod
827     def get_string(line):
828         """Return the value between double quotes
829
830         >>> RustBuild.get_string('    "devel"   ')
831         'devel'
832         >>> RustBuild.get_string("    'devel'   ")
833         'devel'
834         >>> RustBuild.get_string('devel') is None
835         True
836         >>> RustBuild.get_string('    "devel   ')
837         ''
838         """
839         start = line.find('"')
840         if start != -1:
841             end = start + 1 + line[start + 1:].find('"')
842             return line[start + 1:end]
843         start = line.find('\'')
844         if start != -1:
845             end = start + 1 + line[start + 1:].find('\'')
846             return line[start + 1:end]
847         return None
848
849     @staticmethod
850     def exe_suffix():
851         """Return a suffix for executables"""
852         if sys.platform == 'win32':
853             return '.exe'
854         return ''
855
856     def bootstrap_binary(self):
857         """Return the path of the bootstrap binary
858
859         >>> rb = RustBuild()
860         >>> rb.build_dir = "build"
861         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
862         ... "debug", "bootstrap")
863         True
864         """
865         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
866
867     def build_bootstrap(self):
868         """Build bootstrap"""
869         build_dir = os.path.join(self.build_dir, "bootstrap")
870         if self.clean and os.path.exists(build_dir):
871             shutil.rmtree(build_dir)
872         env = os.environ.copy()
873         # `CARGO_BUILD_TARGET` breaks bootstrap build.
874         # See also: <https://github.com/rust-lang/rust/issues/70208>.
875         if "CARGO_BUILD_TARGET" in env:
876             del env["CARGO_BUILD_TARGET"]
877         env["CARGO_TARGET_DIR"] = build_dir
878         env["RUSTC"] = self.rustc(True)
879         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
880             (os.pathsep + env["LD_LIBRARY_PATH"]) \
881             if "LD_LIBRARY_PATH" in env else ""
882         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
883             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
884             if "DYLD_LIBRARY_PATH" in env else ""
885         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
886             (os.pathsep + env["LIBRARY_PATH"]) \
887             if "LIBRARY_PATH" in env else ""
888         # preserve existing RUSTFLAGS
889         env.setdefault("RUSTFLAGS", "")
890         env["RUSTFLAGS"] += " -Cdebuginfo=2"
891
892         build_section = "target.{}".format(self.build)
893         target_features = []
894         if self.get_toml("crt-static", build_section) == "true":
895             target_features += ["+crt-static"]
896         elif self.get_toml("crt-static", build_section) == "false":
897             target_features += ["-crt-static"]
898         if target_features:
899             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
900         target_linker = self.get_toml("linker", build_section)
901         if target_linker is not None:
902             env["RUSTFLAGS"] += " -C linker=" + target_linker
903         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
904         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
905         if self.get_toml("deny-warnings", "rust") != "false":
906             env["RUSTFLAGS"] += " -Dwarnings"
907
908         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
909             os.pathsep + env["PATH"]
910         if not os.path.isfile(self.cargo()):
911             raise Exception("no cargo executable found at `{}`".format(
912                 self.cargo()))
913         args = [self.cargo(), "build", "--manifest-path",
914                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
915         for _ in range(1, self.verbose):
916             args.append("--verbose")
917         if self.use_locked_deps:
918             args.append("--locked")
919         if self.use_vendored_sources:
920             args.append("--frozen")
921         run(args, env=env, verbose=self.verbose)
922
923     def build_triple(self):
924         """Build triple as in LLVM
925
926         Note that `default_build_triple` is moderately expensive,
927         so use `self.build` where possible.
928         """
929         config = self.get_toml('build')
930         if config:
931             return config
932         return default_build_triple(self.verbose)
933
934     def check_submodule(self, module, slow_submodules):
935         if not slow_submodules:
936             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
937                                            cwd=os.path.join(self.rust_root, module),
938                                            stdout=subprocess.PIPE)
939             return checked_out
940         else:
941             return None
942
943     def update_submodule(self, module, checked_out, recorded_submodules):
944         module_path = os.path.join(self.rust_root, module)
945
946         if checked_out is not None:
947             default_encoding = sys.getdefaultencoding()
948             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
949             if recorded_submodules[module] == checked_out:
950                 return
951
952         print("Updating submodule", module)
953
954         run(["git", "submodule", "-q", "sync", module],
955             cwd=self.rust_root, verbose=self.verbose)
956
957         update_args = ["git", "submodule", "update", "--init", "--recursive"]
958         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
959             update_args.append("--progress")
960         update_args.append(module)
961         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
962
963         run(["git", "reset", "-q", "--hard"],
964             cwd=module_path, verbose=self.verbose)
965         run(["git", "clean", "-qdfx"],
966             cwd=module_path, verbose=self.verbose)
967
968     def update_submodules(self):
969         """Update submodules"""
970         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
971                 self.get_toml('submodules') == "false":
972             return
973
974         default_encoding = sys.getdefaultencoding()
975
976         # check the existence and version of 'git' command
977         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
978         self.git_version = distutils.version.LooseVersion(git_version_str)
979
980         slow_submodules = self.get_toml('fast-submodules') == "false"
981         start_time = time()
982         if slow_submodules:
983             print('Unconditionally updating all submodules')
984         else:
985             print('Updating only changed submodules')
986         default_encoding = sys.getdefaultencoding()
987         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
988             ["git", "config", "--file",
989              os.path.join(self.rust_root, ".gitmodules"),
990              "--get-regexp", "path"]
991         ).decode(default_encoding).splitlines()]
992         filtered_submodules = []
993         submodules_names = []
994         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
995         external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
996         llvm_needed = not self.get_toml('codegen-backends', 'rust') \
997             or "llvm" in self.get_toml('codegen-backends', 'rust')
998         for module in submodules:
999             if module.endswith("llvm-project"):
1000                 # Don't sync the llvm-project submodule if an external LLVM was
1001                 # provided, if we are downloading LLVM or if the LLVM backend is
1002                 # not being built. Also, if the submodule has been initialized
1003                 # already, sync it anyways so that it doesn't mess up contributor
1004                 # pull requests.
1005                 if external_llvm_provided or not llvm_needed:
1006                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
1007                         continue
1008             check = self.check_submodule(module, slow_submodules)
1009             filtered_submodules.append((module, check))
1010             submodules_names.append(module)
1011         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1012                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1013         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1014         recorded_submodules = {}
1015         for data in recorded:
1016             data = data.split()
1017             recorded_submodules[data[3]] = data[2]
1018         for module in filtered_submodules:
1019             self.update_submodule(module[0], module[1], recorded_submodules)
1020         print("Submodules updated in %.2f seconds" % (time() - start_time))
1021
1022     def set_normal_environment(self):
1023         """Set download URL for normal environment"""
1024         if 'RUSTUP_DIST_SERVER' in os.environ:
1025             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1026         else:
1027             self._download_url = 'https://static.rust-lang.org'
1028
1029     def set_dev_environment(self):
1030         """Set download URL for development environment"""
1031         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1032             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1033         else:
1034             self._download_url = 'https://dev-static.rust-lang.org'
1035
1036     def check_vendored_status(self):
1037         """Check that vendoring is configured properly"""
1038         vendor_dir = os.path.join(self.rust_root, 'vendor')
1039         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1040             if os.environ.get('USER') != os.environ['SUDO_USER']:
1041                 self.use_vendored_sources = True
1042                 print('info: looks like you are running this command under `sudo`')
1043                 print('      and so in order to preserve your $HOME this will now')
1044                 print('      use vendored sources by default.')
1045                 if not os.path.exists(vendor_dir):
1046                     print('error: vendoring required, but vendor directory does not exist.')
1047                     print('       Run `cargo vendor` without sudo to initialize the '
1048                           'vendor directory.')
1049                     raise Exception("{} not found".format(vendor_dir))
1050
1051         if self.use_vendored_sources:
1052             if not os.path.exists('.cargo'):
1053                 os.makedirs('.cargo')
1054             with output('.cargo/config') as cargo_config:
1055                 cargo_config.write(
1056                     "[source.crates-io]\n"
1057                     "replace-with = 'vendored-sources'\n"
1058                     "registry = 'https://example.com'\n"
1059                     "\n"
1060                     "[source.vendored-sources]\n"
1061                     "directory = '{}/vendor'\n"
1062                     .format(self.rust_root))
1063         else:
1064             if os.path.exists('.cargo'):
1065                 shutil.rmtree('.cargo')
1066
1067     def ensure_vendored(self):
1068         """Ensure that the vendored sources are available if needed"""
1069         vendor_dir = os.path.join(self.rust_root, 'vendor')
1070         # Note that this does not handle updating the vendored dependencies if
1071         # the rust git repository is updated. Normal development usually does
1072         # not use vendoring, so hopefully this isn't too much of a problem.
1073         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1074             run([
1075                 self.cargo(),
1076                 "vendor",
1077                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1078                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1079             ], verbose=self.verbose, cwd=self.rust_root)
1080
1081
1082 def bootstrap(help_triggered):
1083     """Configure, fetch, build and run the initial bootstrap"""
1084
1085     # If the user is asking for help, let them know that the whole download-and-build
1086     # process has to happen before anything is printed out.
1087     if help_triggered:
1088         print("info: Downloading and building bootstrap before processing --help")
1089         print("      command. See src/bootstrap/README.md for help with common")
1090         print("      commands.")
1091
1092     parser = argparse.ArgumentParser(description='Build rust')
1093     parser.add_argument('--config')
1094     parser.add_argument('--build')
1095     parser.add_argument('--clean', action='store_true')
1096     parser.add_argument('-v', '--verbose', action='count', default=0)
1097
1098     args = [a for a in sys.argv if a != '-h' and a != '--help']
1099     args, _ = parser.parse_known_args(args)
1100
1101     # Configure initial bootstrap
1102     build = RustBuild()
1103     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1104     build.verbose = args.verbose
1105     build.clean = args.clean
1106
1107     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1108     # exists).
1109     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1110     if not toml_path and os.path.exists('config.toml'):
1111         toml_path = 'config.toml'
1112
1113     if toml_path:
1114         if not os.path.exists(toml_path):
1115             toml_path = os.path.join(build.rust_root, toml_path)
1116
1117         with open(toml_path) as config:
1118             build.config_toml = config.read()
1119
1120     profile = build.get_toml('profile')
1121     if profile is not None:
1122         include_file = 'config.{}.toml'.format(profile)
1123         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1124         include_path = os.path.join(include_dir, include_file)
1125         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1126         # specific key, so appending our defaults at the end allows the user to override them
1127         with open(include_path) as included_toml:
1128             build.config_toml += os.linesep + included_toml.read()
1129
1130     config_verbose = build.get_toml('verbose', 'build')
1131     if config_verbose is not None:
1132         build.verbose = max(build.verbose, int(config_verbose))
1133
1134     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1135
1136     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1137
1138     build.check_vendored_status()
1139
1140     build_dir = build.get_toml('build-dir', 'build') or 'build'
1141     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1142
1143     data = stage0_data(build.rust_root)
1144     build.date = data['date']
1145     build.rustc_channel = data['rustc']
1146
1147     if "rustfmt" in data:
1148         build.rustfmt_channel = data['rustfmt']
1149
1150     if 'dev' in data:
1151         build.set_dev_environment()
1152     else:
1153         build.set_normal_environment()
1154
1155     build.build = args.build or build.build_triple()
1156     build.update_submodules()
1157
1158     # Fetch/build the bootstrap
1159     build.download_toolchain()
1160     # Download the master compiler if `download-rustc` is set
1161     build.maybe_download_ci_toolchain()
1162     sys.stdout.flush()
1163     build.ensure_vendored()
1164     build.build_bootstrap()
1165     sys.stdout.flush()
1166
1167     # Run the bootstrap
1168     args = [build.bootstrap_binary()]
1169     args.extend(sys.argv[1:])
1170     env = os.environ.copy()
1171     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1172     env["BOOTSTRAP_PYTHON"] = sys.executable
1173     env["BUILD_DIR"] = build.build_dir
1174     env["RUSTC_BOOTSTRAP"] = '1'
1175     if toml_path:
1176         env["BOOTSTRAP_CONFIG"] = toml_path
1177     if build.rustc_commit is not None:
1178         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1179     run(args, env=env, verbose=build.verbose)
1180
1181
1182 def main():
1183     """Entry point for the bootstrap process"""
1184     start_time = time()
1185
1186     # x.py help <cmd> ...
1187     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1188         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1189
1190     help_triggered = (
1191         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1192     try:
1193         bootstrap(help_triggered)
1194         if not help_triggered:
1195             print("Build completed successfully in {}".format(
1196                 format_build_time(time() - start_time)))
1197     except (SystemExit, KeyboardInterrupt) as error:
1198         if hasattr(error, 'code') and isinstance(error.code, int):
1199             exit_code = error.code
1200         else:
1201             exit_code = 1
1202             print(error)
1203         if not help_triggered:
1204             print("Build completed unsuccessfully in {}".format(
1205                 format_build_time(time() - start_time)))
1206         sys.exit(exit_code)
1207
1208
1209 if __name__ == '__main__':
1210     main()