]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
Rollup merge of #82091 - henryboisdequin:use-place-ref-more, r=RalfJung
[rust.git] / src / bootstrap / bootstrap.py
1 from __future__ import absolute_import, division, print_function
2 import argparse
3 import contextlib
4 import datetime
5 import distutils.version
6 import hashlib
7 import os
8 import re
9 import shutil
10 import subprocess
11 import sys
12 import tarfile
13 import tempfile
14
15 from time import time
16
17 def support_xz():
18     try:
19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20             temp_path = temp_file.name
21         with tarfile.open(temp_path, "w:xz"):
22             pass
23         return True
24     except tarfile.CompressionError:
25         return False
26
27 def get(url, path, verbose=False, do_verify=True):
28     suffix = '.sha256'
29     sha_url = url + suffix
30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31         temp_path = temp_file.name
32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33         sha_path = sha_file.name
34
35     try:
36         if do_verify:
37             download(sha_path, sha_url, False, verbose)
38             if os.path.exists(path):
39                 if verify(path, sha_path, False):
40                     if verbose:
41                         print("using already-download file", path)
42                     return
43                 else:
44                     if verbose:
45                         print("ignoring already-download file",
46                             path, "due to failed verification")
47                     os.unlink(path)
48         download(temp_path, url, True, verbose)
49         if do_verify and not verify(temp_path, sha_path, verbose):
50             raise RuntimeError("failed verification")
51         if verbose:
52             print("moving {} to {}".format(temp_path, path))
53         shutil.move(temp_path, path)
54     finally:
55         delete_if_present(sha_path, verbose)
56         delete_if_present(temp_path, verbose)
57
58
59 def delete_if_present(path, verbose):
60     """Remove the given file if present"""
61     if os.path.isfile(path):
62         if verbose:
63             print("removing", path)
64         os.unlink(path)
65
66
67 def download(path, url, probably_big, verbose):
68     for _ in range(0, 4):
69         try:
70             _download(path, url, probably_big, verbose, True)
71             return
72         except RuntimeError:
73             print("\nspurious failure, trying again")
74     _download(path, url, probably_big, verbose, False)
75
76
77 def _download(path, url, probably_big, verbose, exception):
78     if probably_big or verbose:
79         print("downloading {}".format(url))
80     # see http://serverfault.com/questions/301128/how-to-download
81     if sys.platform == 'win32':
82         run(["PowerShell.exe", "/nologo", "-Command",
83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
85             verbose=verbose,
86             exception=exception)
87     else:
88         if probably_big or verbose:
89             option = "-#"
90         else:
91             option = "-s"
92         require(["curl", "--version"])
93         run(["curl", option,
94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
96              "--retry", "3", "-Sf", "-o", path, url],
97             verbose=verbose,
98             exception=exception)
99
100
101 def verify(path, sha_path, verbose):
102     """Check if the sha256 sum of the given path is valid"""
103     if verbose:
104         print("verifying", path)
105     with open(path, "rb") as source:
106         found = hashlib.sha256(source.read()).hexdigest()
107     with open(sha_path, "r") as sha256sum:
108         expected = sha256sum.readline().split()[0]
109     verified = found == expected
110     if not verified:
111         print("invalid checksum:\n"
112               "    found:    {}\n"
113               "    expected: {}".format(found, expected))
114     return verified
115
116
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118     """Unpack the given tarball file"""
119     print("extracting", tarball)
120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
121     with contextlib.closing(tarfile.open(tarball)) as tar:
122         for member in tar.getnames():
123             if "/" not in member:
124                 continue
125             name = member.replace(fname + "/", "", 1)
126             if match is not None and not name.startswith(match):
127                 continue
128             name = name[len(match) + 1:]
129
130             dst_path = os.path.join(dst, name)
131             if verbose:
132                 print("  extracting", member)
133             tar.extract(member, dst)
134             src_path = os.path.join(dst, member)
135             if os.path.isdir(src_path) and os.path.exists(dst_path):
136                 continue
137             shutil.move(src_path, dst_path)
138     shutil.rmtree(os.path.join(dst, fname))
139
140
141 def run(args, verbose=False, exception=False, **kwargs):
142     """Run a child program in a new process"""
143     if verbose:
144         print("running: " + ' '.join(args))
145     sys.stdout.flush()
146     # Use Popen here instead of call() as it apparently allows powershell on
147     # Windows to not lock up waiting for input presumably.
148     ret = subprocess.Popen(args, **kwargs)
149     code = ret.wait()
150     if code != 0:
151         err = "failed to run: " + ' '.join(args)
152         if verbose or exception:
153             raise RuntimeError(err)
154         sys.exit(err)
155
156
157 def require(cmd, exit=True):
158     '''Run a command, returning its output.
159     On error,
160         If `exit` is `True`, exit the process.
161         Otherwise, return None.'''
162     try:
163         return subprocess.check_output(cmd).strip()
164     except (subprocess.CalledProcessError, OSError) as exc:
165         if not exit:
166             return None
167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168         print("Please make sure it's installed and in the path.")
169         sys.exit(1)
170
171
172 def stage0_data(rust_root):
173     """Build a dictionary from stage0.txt"""
174     nightlies = os.path.join(rust_root, "src/stage0.txt")
175     with open(nightlies, 'r') as nightlies:
176         lines = [line.rstrip() for line in nightlies
177                  if not line.startswith("#")]
178         return dict([line.split(": ", 1) for line in lines if line])
179
180
181 def format_build_time(duration):
182     """Return a nicer format for build time
183
184     >>> format_build_time('300')
185     '0:05:00'
186     """
187     return str(datetime.timedelta(seconds=int(duration)))
188
189
190 def default_build_triple(verbose):
191     """Build triple as in LLVM"""
192     # If the user already has a host build triple with an existing `rustc`
193     # install, use their preference. This fixes most issues with Windows builds
194     # being detected as GNU instead of MSVC.
195     default_encoding = sys.getdefaultencoding()
196     try:
197         version = subprocess.check_output(["rustc", "--version", "--verbose"],
198                 stderr=subprocess.DEVNULL)
199         version = version.decode(default_encoding)
200         host = next(x for x in version.split('\n') if x.startswith("host: "))
201         triple = host.split("host: ")[1]
202         if verbose:
203             print("detected default triple {}".format(triple))
204         return triple
205     except Exception as e:
206         if verbose:
207             print("rustup not detected: {}".format(e))
208             print("falling back to auto-detect")
209
210     required = sys.platform != 'win32'
211     ostype = require(["uname", "-s"], exit=required)
212     cputype = require(['uname', '-m'], exit=required)
213
214     # If we do not have `uname`, assume Windows.
215     if ostype is None or cputype is None:
216         return 'x86_64-pc-windows-msvc'
217
218     ostype = ostype.decode(default_encoding)
219     cputype = cputype.decode(default_encoding)
220
221     # The goal here is to come up with the same triple as LLVM would,
222     # at least for the subset of platforms we're willing to target.
223     ostype_mapper = {
224         'Darwin': 'apple-darwin',
225         'DragonFly': 'unknown-dragonfly',
226         'FreeBSD': 'unknown-freebsd',
227         'Haiku': 'unknown-haiku',
228         'NetBSD': 'unknown-netbsd',
229         'OpenBSD': 'unknown-openbsd'
230     }
231
232     # Consider the direct transformation first and then the special cases
233     if ostype in ostype_mapper:
234         ostype = ostype_mapper[ostype]
235     elif ostype == 'Linux':
236         os_from_sp = subprocess.check_output(
237             ['uname', '-o']).strip().decode(default_encoding)
238         if os_from_sp == 'Android':
239             ostype = 'linux-android'
240         else:
241             ostype = 'unknown-linux-gnu'
242     elif ostype == 'SunOS':
243         ostype = 'sun-solaris'
244         # On Solaris, uname -m will return a machine classification instead
245         # of a cpu type, so uname -p is recommended instead.  However, the
246         # output from that option is too generic for our purposes (it will
247         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
248         # must be used instead.
249         cputype = require(['isainfo', '-k']).decode(default_encoding)
250     elif ostype.startswith('MINGW'):
251         # msys' `uname` does not print gcc configuration, but prints msys
252         # configuration. so we cannot believe `uname -m`:
253         # msys1 is always i686 and msys2 is always x86_64.
254         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
255         # MINGW64 on x86_64.
256         ostype = 'pc-windows-gnu'
257         cputype = 'i686'
258         if os.environ.get('MSYSTEM') == 'MINGW64':
259             cputype = 'x86_64'
260     elif ostype.startswith('MSYS'):
261         ostype = 'pc-windows-gnu'
262     elif ostype.startswith('CYGWIN_NT'):
263         cputype = 'i686'
264         if ostype.endswith('WOW64'):
265             cputype = 'x86_64'
266         ostype = 'pc-windows-gnu'
267     elif sys.platform == 'win32':
268         # Some Windows platforms might have a `uname` command that returns a
269         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
270         # these cases, fall back to using sys.platform.
271         return 'x86_64-pc-windows-msvc'
272     else:
273         err = "unknown OS type: {}".format(ostype)
274         sys.exit(err)
275
276     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
277         cputype = subprocess.check_output(
278               ['uname', '-p']).strip().decode(default_encoding)
279     cputype_mapper = {
280         'BePC': 'i686',
281         'aarch64': 'aarch64',
282         'amd64': 'x86_64',
283         'arm64': 'aarch64',
284         'i386': 'i686',
285         'i486': 'i686',
286         'i686': 'i686',
287         'i786': 'i686',
288         'powerpc': 'powerpc',
289         'powerpc64': 'powerpc64',
290         'powerpc64le': 'powerpc64le',
291         'ppc': 'powerpc',
292         'ppc64': 'powerpc64',
293         'ppc64le': 'powerpc64le',
294         's390x': 's390x',
295         'x64': 'x86_64',
296         'x86': 'i686',
297         'x86-64': 'x86_64',
298         'x86_64': 'x86_64'
299     }
300
301     # Consider the direct transformation first and then the special cases
302     if cputype in cputype_mapper:
303         cputype = cputype_mapper[cputype]
304     elif cputype in {'xscale', 'arm'}:
305         cputype = 'arm'
306         if ostype == 'linux-android':
307             ostype = 'linux-androideabi'
308         elif ostype == 'unknown-freebsd':
309             cputype = subprocess.check_output(
310                 ['uname', '-p']).strip().decode(default_encoding)
311             ostype = 'unknown-freebsd'
312     elif cputype == 'armv6l':
313         cputype = 'arm'
314         if ostype == 'linux-android':
315             ostype = 'linux-androideabi'
316         else:
317             ostype += 'eabihf'
318     elif cputype in {'armv7l', 'armv8l'}:
319         cputype = 'armv7'
320         if ostype == 'linux-android':
321             ostype = 'linux-androideabi'
322         else:
323             ostype += 'eabihf'
324     elif cputype == 'mips':
325         if sys.byteorder == 'big':
326             cputype = 'mips'
327         elif sys.byteorder == 'little':
328             cputype = 'mipsel'
329         else:
330             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
331     elif cputype == 'mips64':
332         if sys.byteorder == 'big':
333             cputype = 'mips64'
334         elif sys.byteorder == 'little':
335             cputype = 'mips64el'
336         else:
337             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
338         # only the n64 ABI is supported, indicate it
339         ostype += 'abi64'
340     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
341         pass
342     else:
343         err = "unknown cpu type: {}".format(cputype)
344         sys.exit(err)
345
346     return "{}-{}".format(cputype, ostype)
347
348
349 @contextlib.contextmanager
350 def output(filepath):
351     tmp = filepath + '.tmp'
352     with open(tmp, 'w') as f:
353         yield f
354     try:
355         if os.path.exists(filepath):
356             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
357     except OSError:
358         shutil.copy2(tmp, filepath)
359         os.remove(tmp)
360         return
361     os.rename(tmp, filepath)
362
363
364 class RustBuild(object):
365     """Provide all the methods required to build Rust"""
366     def __init__(self):
367         self.date = ''
368         self._download_url = ''
369         self.rustc_channel = ''
370         self.rustfmt_channel = ''
371         self.build = ''
372         self.build_dir = ''
373         self.clean = False
374         self.config_toml = ''
375         self.rust_root = ''
376         self.use_locked_deps = ''
377         self.use_vendored_sources = ''
378         self.verbose = False
379         self.git_version = None
380         self.nix_deps_dir = None
381         self.rustc_commit = None
382
383     def download_stage0(self):
384         """Fetch the build system for Rust, written in Rust
385
386         This method will build a cache directory, then it will fetch the
387         tarball which has the stage0 compiler used to then bootstrap the Rust
388         compiler itself.
389
390         Each downloaded tarball is extracted, after that, the script
391         will move all the content to the right place.
392         """
393         rustc_channel = self.rustc_channel
394         rustfmt_channel = self.rustfmt_channel
395
396         if self.rustc().startswith(self.bin_root()) and \
397                 (not os.path.exists(self.rustc()) or
398                  self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
399             if os.path.exists(self.bin_root()):
400                 shutil.rmtree(self.bin_root())
401             download_rustc = self.rustc_commit is not None
402             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
403             filename = "rust-std-{}-{}{}".format(
404                 rustc_channel, self.build, tarball_suffix)
405             pattern = "rust-std-{}".format(self.build)
406             self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
407             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
408                                               tarball_suffix)
409             self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
410             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
411                                               tarball_suffix)
412             self._download_component_helper(filename, "cargo", tarball_suffix)
413             if self.rustc_commit is not None:
414                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
415                 self._download_component_helper(
416                     filename, "rustc-dev", tarball_suffix, download_rustc
417                 )
418
419             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
420             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
421             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
422             lib_dir = "{}/lib".format(self.bin_root())
423             for lib in os.listdir(lib_dir):
424                 if lib.endswith(".so"):
425                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
426             with output(self.rustc_stamp()) as rust_stamp:
427                 rust_stamp.write(self.date + str(self.rustc_commit))
428
429         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
430             not os.path.exists(self.rustfmt())
431             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
432         ):
433             if rustfmt_channel:
434                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
435                 [channel, date] = rustfmt_channel.split('-', 1)
436                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
437                 self._download_component_helper(
438                     filename, "rustfmt-preview", tarball_suffix, key=date
439                 )
440                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
441                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
442                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
443                     rustfmt_stamp.write(self.rustfmt_channel)
444
445         if self.downloading_llvm():
446             # We want the most recent LLVM submodule update to avoid downloading
447             # LLVM more often than necessary.
448             #
449             # This git command finds that commit SHA, looking for bors-authored
450             # merges that modified src/llvm-project.
451             #
452             # This works even in a repository that has not yet initialized
453             # submodules.
454             top_level = subprocess.check_output([
455                 "git", "rev-parse", "--show-toplevel",
456             ]).decode(sys.getdefaultencoding()).strip()
457             llvm_sha = subprocess.check_output([
458                 "git", "log", "--author=bors", "--format=%H", "-n1",
459                 "-m", "--first-parent",
460                 "--",
461                 "{}/src/llvm-project".format(top_level),
462                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
463             ]).decode(sys.getdefaultencoding()).strip()
464             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
465             llvm_root = self.llvm_root()
466             llvm_lib = os.path.join(llvm_root, "lib")
467             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
468                 self._download_ci_llvm(llvm_sha, llvm_assertions)
469                 for binary in ["llvm-config", "FileCheck"]:
470                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary), rpath_libz=True)
471                 for lib in os.listdir(llvm_lib):
472                     if lib.endswith(".so"):
473                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib), rpath_libz=True)
474                 with output(self.llvm_stamp()) as llvm_stamp:
475                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
476
477     def downloading_llvm(self):
478         opt = self.get_toml('download-ci-llvm', 'llvm')
479         # This is currently all tier 1 targets (since others may not have CI
480         # artifacts)
481         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
482         supported_platforms = [
483             "aarch64-unknown-linux-gnu",
484             "i686-pc-windows-gnu",
485             "i686-pc-windows-msvc",
486             "i686-unknown-linux-gnu",
487             "x86_64-unknown-linux-gnu",
488             "x86_64-apple-darwin",
489             "x86_64-pc-windows-gnu",
490             "x86_64-pc-windows-msvc",
491         ]
492         return opt == "true" \
493             or (opt == "if-available" and self.build in supported_platforms)
494
495     def _download_component_helper(
496         self, filename, pattern, tarball_suffix, download_rustc=False, key=None
497     ):
498         if key is None:
499             if download_rustc:
500                 key = self.rustc_commit
501             else:
502                 key = self.date
503         cache_dst = os.path.join(self.build_dir, "cache")
504         rustc_cache = os.path.join(cache_dst, key)
505         if not os.path.exists(rustc_cache):
506             os.makedirs(rustc_cache)
507
508         if download_rustc:
509             url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
510         else:
511             url = "{}/dist/{}".format(self._download_url, key)
512         tarball = os.path.join(rustc_cache, filename)
513         if not os.path.exists(tarball):
514             do_verify = not download_rustc
515             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
516         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
517
518     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
519         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
520         cache_dst = os.path.join(self.build_dir, "cache")
521         rustc_cache = os.path.join(cache_dst, cache_prefix)
522         if not os.path.exists(rustc_cache):
523             os.makedirs(rustc_cache)
524
525         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
526         if llvm_assertions:
527             url = url.replace('rustc-builds', 'rustc-builds-alt')
528         # ci-artifacts are only stored as .xz, not .gz
529         if not support_xz():
530             print("error: XZ support is required to download LLVM")
531             print("help: consider disabling `download-ci-llvm` or using python3")
532             exit(1)
533         tarball_suffix = '.tar.xz'
534         filename = "rust-dev-nightly-" + self.build + tarball_suffix
535         tarball = os.path.join(rustc_cache, filename)
536         if not os.path.exists(tarball):
537             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
538         unpack(tarball, tarball_suffix, self.llvm_root(),
539                 match="rust-dev",
540                 verbose=self.verbose)
541
542     def fix_bin_or_dylib(self, fname, rpath_libz=False):
543         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
544         or the RPATH section, to fix the dynamic library search path
545
546         This method is only required on NixOS and uses the PatchELF utility to
547         change the interpreter/RPATH of ELF executables.
548
549         Please see https://nixos.org/patchelf.html for more information
550         """
551         default_encoding = sys.getdefaultencoding()
552         try:
553             ostype = subprocess.check_output(
554                 ['uname', '-s']).strip().decode(default_encoding)
555         except subprocess.CalledProcessError:
556             return
557         except OSError as reason:
558             if getattr(reason, 'winerror', None) is not None:
559                 return
560             raise reason
561
562         if ostype != "Linux":
563             return
564
565         if not os.path.exists("/etc/NIXOS"):
566             return
567         if os.path.exists("/lib"):
568             return
569
570         # At this point we're pretty sure the user is running NixOS
571         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
572         print(nix_os_msg, fname)
573
574         # Only build `stage0/.nix-deps` once.
575         nix_deps_dir = self.nix_deps_dir
576         if not nix_deps_dir:
577             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
578             if not os.path.exists(nix_deps_dir):
579                 os.makedirs(nix_deps_dir)
580
581             nix_deps = [
582                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
583                 "stdenv.cc.bintools",
584
585                 # Needed as a system dependency of `libLLVM-*.so`.
586                 "zlib",
587
588                 # Needed for patching ELF binaries (see doc comment above).
589                 "patchelf",
590             ]
591
592             # Run `nix-build` to "build" each dependency (which will likely reuse
593             # the existing `/nix/store` copy, or at most download a pre-built copy).
594             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
595             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
596             # ensuring garbage collection will never remove the `/nix/store` path
597             # (which would break our patched binaries that hardcode those paths).
598             for dep in nix_deps:
599                 try:
600                     subprocess.check_output([
601                         "nix-build", "<nixpkgs>",
602                         "-A", dep,
603                         "-o", "{}/{}".format(nix_deps_dir, dep),
604                     ])
605                 except subprocess.CalledProcessError as reason:
606                     print("warning: failed to call nix-build:", reason)
607                     return
608
609             self.nix_deps_dir = nix_deps_dir
610
611         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
612         patchelf_args = []
613
614         if rpath_libz:
615             # Patch RPATH to add `zlib` dependency that stems from LLVM
616             dylib_deps = ["zlib"]
617             rpath_entries = [
618                 # Relative default, all binary and dynamic libraries we ship
619                 # appear to have this (even when `../lib` is redundant).
620                 "$ORIGIN/../lib",
621             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
622             patchelf_args += ["--set-rpath", ":".join(rpath_entries)]
623         if not fname.endswith(".so"):
624             # Finally, set the corret .interp for binaries
625             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
626             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
627                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
628
629         try:
630             subprocess.check_output([patchelf] + patchelf_args + [fname])
631         except subprocess.CalledProcessError as reason:
632             print("warning: failed to call patchelf:", reason)
633             return
634
635     # Return the stage1 compiler to download, if any.
636     def maybe_download_rustc(self):
637         # If `download-rustc` is not set, default to rebuilding.
638         if self.get_toml("download-rustc", section="rust") != "true":
639             return None
640
641         # Handle running from a directory other than the top level
642         rev_parse = ["git", "rev-parse", "--show-toplevel"]
643         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
644         compiler = "{}/compiler/".format(top_level)
645
646         # Look for a version to compare to based on the current commit.
647         # Ideally this would just use `merge-base`, but on beta and stable branches that wouldn't
648         # come up with any commits, so hack it and use `author=bors` instead.
649         merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1", "--", compiler]
650         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
651
652         # Warn if there were changes to the compiler since the ancestor commit.
653         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
654         if status != 0:
655             print("warning: `download-rustc` is enabled, but there are changes to compiler/")
656
657         return commit
658
659     def rustc_stamp(self):
660         """Return the path for .rustc-stamp
661
662         >>> rb = RustBuild()
663         >>> rb.build_dir = "build"
664         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
665         True
666         """
667         return os.path.join(self.bin_root(), '.rustc-stamp')
668
669     def rustfmt_stamp(self):
670         """Return the path for .rustfmt-stamp
671
672         >>> rb = RustBuild()
673         >>> rb.build_dir = "build"
674         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
675         True
676         """
677         return os.path.join(self.bin_root(), '.rustfmt-stamp')
678
679     def llvm_stamp(self):
680         """Return the path for .rustfmt-stamp
681
682         >>> rb = RustBuild()
683         >>> rb.build_dir = "build"
684         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
685         True
686         """
687         return os.path.join(self.llvm_root(), '.llvm-stamp')
688
689
690     def program_out_of_date(self, stamp_path, key):
691         """Check if the given program stamp is out of date"""
692         if not os.path.exists(stamp_path) or self.clean:
693             return True
694         with open(stamp_path, 'r') as stamp:
695             return key != stamp.read()
696
697     def bin_root(self):
698         """Return the binary root directory
699
700         >>> rb = RustBuild()
701         >>> rb.build_dir = "build"
702         >>> rb.bin_root() == os.path.join("build", "stage0")
703         True
704
705         When the 'build' property is given should be a nested directory:
706
707         >>> rb.build = "devel"
708         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
709         True
710         """
711         return os.path.join(self.build_dir, self.build, "stage0")
712
713     def llvm_root(self):
714         """Return the CI LLVM root directory
715
716         >>> rb = RustBuild()
717         >>> rb.build_dir = "build"
718         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
719         True
720
721         When the 'build' property is given should be a nested directory:
722
723         >>> rb.build = "devel"
724         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
725         True
726         """
727         return os.path.join(self.build_dir, self.build, "ci-llvm")
728
729     def get_toml(self, key, section=None):
730         """Returns the value of the given key in config.toml, otherwise returns None
731
732         >>> rb = RustBuild()
733         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
734         >>> rb.get_toml("key2")
735         'value2'
736
737         If the key does not exists, the result is None:
738
739         >>> rb.get_toml("key3") is None
740         True
741
742         Optionally also matches the section the key appears in
743
744         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
745         >>> rb.get_toml('key', 'a')
746         'value1'
747         >>> rb.get_toml('key', 'b')
748         'value2'
749         >>> rb.get_toml('key', 'c') is None
750         True
751
752         >>> rb.config_toml = 'key1 = true'
753         >>> rb.get_toml("key1")
754         'true'
755         """
756
757         cur_section = None
758         for line in self.config_toml.splitlines():
759             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
760             if section_match is not None:
761                 cur_section = section_match.group(1)
762
763             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
764             if match is not None:
765                 value = match.group(1)
766                 if section is None or section == cur_section:
767                     return self.get_string(value) or value.strip()
768         return None
769
770     def cargo(self):
771         """Return config path for cargo"""
772         return self.program_config('cargo')
773
774     def rustc(self):
775         """Return config path for rustc"""
776         return self.program_config('rustc')
777
778     def rustfmt(self):
779         """Return config path for rustfmt"""
780         if not self.rustfmt_channel:
781             return None
782         return self.program_config('rustfmt')
783
784     def program_config(self, program):
785         """Return config path for the given program
786
787         >>> rb = RustBuild()
788         >>> rb.config_toml = 'rustc = "rustc"\\n'
789         >>> rb.program_config('rustc')
790         'rustc'
791         >>> rb.config_toml = ''
792         >>> cargo_path = rb.program_config('cargo')
793         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
794         ... "bin", "cargo")
795         True
796         """
797         config = self.get_toml(program)
798         if config:
799             return os.path.expanduser(config)
800         return os.path.join(self.bin_root(), "bin", "{}{}".format(
801             program, self.exe_suffix()))
802
803     @staticmethod
804     def get_string(line):
805         """Return the value between double quotes
806
807         >>> RustBuild.get_string('    "devel"   ')
808         'devel'
809         >>> RustBuild.get_string("    'devel'   ")
810         'devel'
811         >>> RustBuild.get_string('devel') is None
812         True
813         >>> RustBuild.get_string('    "devel   ')
814         ''
815         """
816         start = line.find('"')
817         if start != -1:
818             end = start + 1 + line[start + 1:].find('"')
819             return line[start + 1:end]
820         start = line.find('\'')
821         if start != -1:
822             end = start + 1 + line[start + 1:].find('\'')
823             return line[start + 1:end]
824         return None
825
826     @staticmethod
827     def exe_suffix():
828         """Return a suffix for executables"""
829         if sys.platform == 'win32':
830             return '.exe'
831         return ''
832
833     def bootstrap_binary(self):
834         """Return the path of the bootstrap binary
835
836         >>> rb = RustBuild()
837         >>> rb.build_dir = "build"
838         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
839         ... "debug", "bootstrap")
840         True
841         """
842         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
843
844     def build_bootstrap(self):
845         """Build bootstrap"""
846         build_dir = os.path.join(self.build_dir, "bootstrap")
847         if self.clean and os.path.exists(build_dir):
848             shutil.rmtree(build_dir)
849         env = os.environ.copy()
850         # `CARGO_BUILD_TARGET` breaks bootstrap build.
851         # See also: <https://github.com/rust-lang/rust/issues/70208>.
852         if "CARGO_BUILD_TARGET" in env:
853             del env["CARGO_BUILD_TARGET"]
854         env["CARGO_TARGET_DIR"] = build_dir
855         env["RUSTC"] = self.rustc()
856         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
857             (os.pathsep + env["LD_LIBRARY_PATH"]) \
858             if "LD_LIBRARY_PATH" in env else ""
859         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
860             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
861             if "DYLD_LIBRARY_PATH" in env else ""
862         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
863             (os.pathsep + env["LIBRARY_PATH"]) \
864             if "LIBRARY_PATH" in env else ""
865         # preserve existing RUSTFLAGS
866         env.setdefault("RUSTFLAGS", "")
867         env["RUSTFLAGS"] += " -Cdebuginfo=2"
868
869         build_section = "target.{}".format(self.build)
870         target_features = []
871         if self.get_toml("crt-static", build_section) == "true":
872             target_features += ["+crt-static"]
873         elif self.get_toml("crt-static", build_section) == "false":
874             target_features += ["-crt-static"]
875         if target_features:
876             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
877         target_linker = self.get_toml("linker", build_section)
878         if target_linker is not None:
879             env["RUSTFLAGS"] += " -C linker=" + target_linker
880         # cfg(bootstrap): Add `-Wsemicolon_in_expressions_from_macros` after the next beta bump
881         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
882         if self.get_toml("deny-warnings", "rust") != "false":
883             env["RUSTFLAGS"] += " -Dwarnings"
884
885         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
886             os.pathsep + env["PATH"]
887         if not os.path.isfile(self.cargo()):
888             raise Exception("no cargo executable found at `{}`".format(
889                 self.cargo()))
890         args = [self.cargo(), "build", "--manifest-path",
891                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
892         for _ in range(1, self.verbose):
893             args.append("--verbose")
894         if self.use_locked_deps:
895             args.append("--locked")
896         if self.use_vendored_sources:
897             args.append("--frozen")
898         run(args, env=env, verbose=self.verbose)
899
900     def build_triple(self):
901         """Build triple as in LLVM
902
903         Note that `default_build_triple` is moderately expensive,
904         so use `self.build` where possible.
905         """
906         config = self.get_toml('build')
907         if config:
908             return config
909         return default_build_triple(self.verbose)
910
911     def check_submodule(self, module, slow_submodules):
912         if not slow_submodules:
913             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
914                                            cwd=os.path.join(self.rust_root, module),
915                                            stdout=subprocess.PIPE)
916             return checked_out
917         else:
918             return None
919
920     def update_submodule(self, module, checked_out, recorded_submodules):
921         module_path = os.path.join(self.rust_root, module)
922
923         if checked_out is not None:
924             default_encoding = sys.getdefaultencoding()
925             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
926             if recorded_submodules[module] == checked_out:
927                 return
928
929         print("Updating submodule", module)
930
931         run(["git", "submodule", "-q", "sync", module],
932             cwd=self.rust_root, verbose=self.verbose)
933
934         update_args = ["git", "submodule", "update", "--init", "--recursive"]
935         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
936             update_args.append("--progress")
937         update_args.append(module)
938         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
939
940         run(["git", "reset", "-q", "--hard"],
941             cwd=module_path, verbose=self.verbose)
942         run(["git", "clean", "-qdfx"],
943             cwd=module_path, verbose=self.verbose)
944
945     def update_submodules(self):
946         """Update submodules"""
947         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
948                 self.get_toml('submodules') == "false":
949             return
950
951         default_encoding = sys.getdefaultencoding()
952
953         # check the existence and version of 'git' command
954         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
955         self.git_version = distutils.version.LooseVersion(git_version_str)
956
957         slow_submodules = self.get_toml('fast-submodules') == "false"
958         start_time = time()
959         if slow_submodules:
960             print('Unconditionally updating all submodules')
961         else:
962             print('Updating only changed submodules')
963         default_encoding = sys.getdefaultencoding()
964         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
965             ["git", "config", "--file",
966              os.path.join(self.rust_root, ".gitmodules"),
967              "--get-regexp", "path"]
968         ).decode(default_encoding).splitlines()]
969         filtered_submodules = []
970         submodules_names = []
971         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
972         external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
973         llvm_needed = not self.get_toml('codegen-backends', 'rust') \
974             or "llvm" in self.get_toml('codegen-backends', 'rust')
975         for module in submodules:
976             if module.endswith("llvm-project"):
977                 # Don't sync the llvm-project submodule if an external LLVM was
978                 # provided, if we are downloading LLVM or if the LLVM backend is
979                 # not being built. Also, if the submodule has been initialized
980                 # already, sync it anyways so that it doesn't mess up contributor
981                 # pull requests.
982                 if external_llvm_provided or not llvm_needed:
983                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
984                         continue
985             check = self.check_submodule(module, slow_submodules)
986             filtered_submodules.append((module, check))
987             submodules_names.append(module)
988         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
989                                     cwd=self.rust_root, stdout=subprocess.PIPE)
990         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
991         recorded_submodules = {}
992         for data in recorded:
993             data = data.split()
994             recorded_submodules[data[3]] = data[2]
995         for module in filtered_submodules:
996             self.update_submodule(module[0], module[1], recorded_submodules)
997         print("Submodules updated in %.2f seconds" % (time() - start_time))
998
999     def set_normal_environment(self):
1000         """Set download URL for normal environment"""
1001         if 'RUSTUP_DIST_SERVER' in os.environ:
1002             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1003         else:
1004             self._download_url = 'https://static.rust-lang.org'
1005
1006     def set_dev_environment(self):
1007         """Set download URL for development environment"""
1008         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1009             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1010         else:
1011             self._download_url = 'https://dev-static.rust-lang.org'
1012
1013     def check_vendored_status(self):
1014         """Check that vendoring is configured properly"""
1015         vendor_dir = os.path.join(self.rust_root, 'vendor')
1016         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1017             if os.environ.get('USER') != os.environ['SUDO_USER']:
1018                 self.use_vendored_sources = True
1019                 print('info: looks like you are running this command under `sudo`')
1020                 print('      and so in order to preserve your $HOME this will now')
1021                 print('      use vendored sources by default.')
1022                 if not os.path.exists(vendor_dir):
1023                     print('error: vendoring required, but vendor directory does not exist.')
1024                     print('       Run `cargo vendor` without sudo to initialize the '
1025                           'vendor directory.')
1026                     raise Exception("{} not found".format(vendor_dir))
1027
1028         if self.use_vendored_sources:
1029             if not os.path.exists('.cargo'):
1030                 os.makedirs('.cargo')
1031             with output('.cargo/config') as cargo_config:
1032                 cargo_config.write(
1033                     "[source.crates-io]\n"
1034                     "replace-with = 'vendored-sources'\n"
1035                     "registry = 'https://example.com'\n"
1036                     "\n"
1037                     "[source.vendored-sources]\n"
1038                     "directory = '{}/vendor'\n"
1039                     .format(self.rust_root))
1040         else:
1041             if os.path.exists('.cargo'):
1042                 shutil.rmtree('.cargo')
1043
1044     def ensure_vendored(self):
1045         """Ensure that the vendored sources are available if needed"""
1046         vendor_dir = os.path.join(self.rust_root, 'vendor')
1047         # Note that this does not handle updating the vendored dependencies if
1048         # the rust git repository is updated. Normal development usually does
1049         # not use vendoring, so hopefully this isn't too much of a problem.
1050         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1051             run([
1052                 self.cargo(),
1053                 "vendor",
1054                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1055                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1056             ], verbose=self.verbose, cwd=self.rust_root)
1057
1058
1059 def bootstrap(help_triggered):
1060     """Configure, fetch, build and run the initial bootstrap"""
1061
1062     # If the user is asking for help, let them know that the whole download-and-build
1063     # process has to happen before anything is printed out.
1064     if help_triggered:
1065         print("info: Downloading and building bootstrap before processing --help")
1066         print("      command. See src/bootstrap/README.md for help with common")
1067         print("      commands.")
1068
1069     parser = argparse.ArgumentParser(description='Build rust')
1070     parser.add_argument('--config')
1071     parser.add_argument('--build')
1072     parser.add_argument('--clean', action='store_true')
1073     parser.add_argument('-v', '--verbose', action='count', default=0)
1074
1075     args = [a for a in sys.argv if a != '-h' and a != '--help']
1076     args, _ = parser.parse_known_args(args)
1077
1078     # Configure initial bootstrap
1079     build = RustBuild()
1080     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1081     build.verbose = args.verbose
1082     build.clean = args.clean
1083
1084     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1085     # exists).
1086     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1087     if not toml_path and os.path.exists('config.toml'):
1088         toml_path = 'config.toml'
1089
1090     if toml_path:
1091         if not os.path.exists(toml_path):
1092             toml_path = os.path.join(build.rust_root, toml_path)
1093
1094         with open(toml_path) as config:
1095             build.config_toml = config.read()
1096
1097     profile = build.get_toml('profile')
1098     if profile is not None:
1099         include_file = 'config.{}.toml'.format(profile)
1100         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1101         include_path = os.path.join(include_dir, include_file)
1102         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1103         # specific key, so appending our defaults at the end allows the user to override them
1104         with open(include_path) as included_toml:
1105             build.config_toml += os.linesep + included_toml.read()
1106
1107     config_verbose = build.get_toml('verbose', 'build')
1108     if config_verbose is not None:
1109         build.verbose = max(build.verbose, int(config_verbose))
1110
1111     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1112
1113     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1114
1115     build.check_vendored_status()
1116
1117     build_dir = build.get_toml('build-dir', 'build') or 'build'
1118     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1119
1120     data = stage0_data(build.rust_root)
1121     build.date = data['date']
1122     build.rustc_channel = data['rustc']
1123
1124     if "rustfmt" in data:
1125         build.rustfmt_channel = data['rustfmt']
1126
1127     if 'dev' in data:
1128         build.set_dev_environment()
1129     else:
1130         build.set_normal_environment()
1131
1132     build.build = args.build or build.build_triple()
1133     build.update_submodules()
1134
1135     # Fetch/build the bootstrap
1136     build.rustc_commit = build.maybe_download_rustc()
1137     if build.rustc_commit is not None:
1138         if build.verbose:
1139             commit = build.rustc_commit
1140             print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
1141         # FIXME: support downloading artifacts from the beta channel
1142         build.rustc_channel = "nightly"
1143     build.download_stage0()
1144     sys.stdout.flush()
1145     build.ensure_vendored()
1146     build.build_bootstrap()
1147     sys.stdout.flush()
1148
1149     # Run the bootstrap
1150     args = [build.bootstrap_binary()]
1151     args.extend(sys.argv[1:])
1152     env = os.environ.copy()
1153     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1154     env["BOOTSTRAP_PYTHON"] = sys.executable
1155     env["BUILD_DIR"] = build.build_dir
1156     env["RUSTC_BOOTSTRAP"] = '1'
1157     if toml_path:
1158         env["BOOTSTRAP_CONFIG"] = toml_path
1159     run(args, env=env, verbose=build.verbose)
1160
1161
1162 def main():
1163     """Entry point for the bootstrap process"""
1164     start_time = time()
1165
1166     # x.py help <cmd> ...
1167     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1168         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1169
1170     help_triggered = (
1171         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1172     try:
1173         bootstrap(help_triggered)
1174         if not help_triggered:
1175             print("Build completed successfully in {}".format(
1176                 format_build_time(time() - start_time)))
1177     except (SystemExit, KeyboardInterrupt) as error:
1178         if hasattr(error, 'code') and isinstance(error.code, int):
1179             exit_code = error.code
1180         else:
1181             exit_code = 1
1182             print(error)
1183         if not help_triggered:
1184             print("Build completed unsuccessfully in {}".format(
1185                 format_build_time(time() - start_time)))
1186         sys.exit(exit_code)
1187
1188
1189 if __name__ == '__main__':
1190     main()