]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
Add support for compile-flags in coverage tests
[rust.git] / src / bootstrap / bootstrap.py
1 from __future__ import absolute_import, division, print_function
2 import argparse
3 import contextlib
4 import datetime
5 import distutils.version
6 import hashlib
7 import os
8 import re
9 import shutil
10 import subprocess
11 import sys
12 import tarfile
13 import tempfile
14
15 from time import time
16
17 def support_xz():
18     try:
19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20             temp_path = temp_file.name
21         with tarfile.open(temp_path, "w:xz"):
22             pass
23         return True
24     except tarfile.CompressionError:
25         return False
26
27 def get(url, path, verbose=False, do_verify=True):
28     suffix = '.sha256'
29     sha_url = url + suffix
30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31         temp_path = temp_file.name
32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33         sha_path = sha_file.name
34
35     try:
36         if do_verify:
37             download(sha_path, sha_url, False, verbose)
38             if os.path.exists(path):
39                 if verify(path, sha_path, False):
40                     if verbose:
41                         print("using already-download file", path)
42                     return
43                 else:
44                     if verbose:
45                         print("ignoring already-download file",
46                             path, "due to failed verification")
47                     os.unlink(path)
48         download(temp_path, url, True, verbose)
49         if do_verify and not verify(temp_path, sha_path, verbose):
50             raise RuntimeError("failed verification")
51         if verbose:
52             print("moving {} to {}".format(temp_path, path))
53         shutil.move(temp_path, path)
54     finally:
55         delete_if_present(sha_path, verbose)
56         delete_if_present(temp_path, verbose)
57
58
59 def delete_if_present(path, verbose):
60     """Remove the given file if present"""
61     if os.path.isfile(path):
62         if verbose:
63             print("removing", path)
64         os.unlink(path)
65
66
67 def download(path, url, probably_big, verbose):
68     for _ in range(0, 4):
69         try:
70             _download(path, url, probably_big, verbose, True)
71             return
72         except RuntimeError:
73             print("\nspurious failure, trying again")
74     _download(path, url, probably_big, verbose, False)
75
76
77 def _download(path, url, probably_big, verbose, exception):
78     if probably_big or verbose:
79         print("downloading {}".format(url))
80     # see http://serverfault.com/questions/301128/how-to-download
81     if sys.platform == 'win32':
82         run(["PowerShell.exe", "/nologo", "-Command",
83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
85             verbose=verbose,
86             exception=exception)
87     else:
88         if probably_big or verbose:
89             option = "-#"
90         else:
91             option = "-s"
92         require(["curl", "--version"])
93         run(["curl", option,
94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
96              "--retry", "3", "-Sf", "-o", path, url],
97             verbose=verbose,
98             exception=exception)
99
100
101 def verify(path, sha_path, verbose):
102     """Check if the sha256 sum of the given path is valid"""
103     if verbose:
104         print("verifying", path)
105     with open(path, "rb") as source:
106         found = hashlib.sha256(source.read()).hexdigest()
107     with open(sha_path, "r") as sha256sum:
108         expected = sha256sum.readline().split()[0]
109     verified = found == expected
110     if not verified:
111         print("invalid checksum:\n"
112               "    found:    {}\n"
113               "    expected: {}".format(found, expected))
114     return verified
115
116
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118     """Unpack the given tarball file"""
119     print("extracting", tarball)
120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
121     with contextlib.closing(tarfile.open(tarball)) as tar:
122         for member in tar.getnames():
123             if "/" not in member:
124                 continue
125             name = member.replace(fname + "/", "", 1)
126             if match is not None and not name.startswith(match):
127                 continue
128             name = name[len(match) + 1:]
129
130             dst_path = os.path.join(dst, name)
131             if verbose:
132                 print("  extracting", member)
133             tar.extract(member, dst)
134             src_path = os.path.join(dst, member)
135             if os.path.isdir(src_path) and os.path.exists(dst_path):
136                 continue
137             shutil.move(src_path, dst_path)
138     shutil.rmtree(os.path.join(dst, fname))
139
140
141 def run(args, verbose=False, exception=False, **kwargs):
142     """Run a child program in a new process"""
143     if verbose:
144         print("running: " + ' '.join(args))
145     sys.stdout.flush()
146     # Use Popen here instead of call() as it apparently allows powershell on
147     # Windows to not lock up waiting for input presumably.
148     ret = subprocess.Popen(args, **kwargs)
149     code = ret.wait()
150     if code != 0:
151         err = "failed to run: " + ' '.join(args)
152         if verbose or exception:
153             raise RuntimeError(err)
154         sys.exit(err)
155
156
157 def require(cmd, exit=True):
158     '''Run a command, returning its output.
159     On error,
160         If `exit` is `True`, exit the process.
161         Otherwise, return None.'''
162     try:
163         return subprocess.check_output(cmd).strip()
164     except (subprocess.CalledProcessError, OSError) as exc:
165         if not exit:
166             return None
167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168         print("Please make sure it's installed and in the path.")
169         sys.exit(1)
170
171
172 def stage0_data(rust_root):
173     """Build a dictionary from stage0.txt"""
174     nightlies = os.path.join(rust_root, "src/stage0.txt")
175     with open(nightlies, 'r') as nightlies:
176         lines = [line.rstrip() for line in nightlies
177                  if not line.startswith("#")]
178         return dict([line.split(": ", 1) for line in lines if line])
179
180
181 def format_build_time(duration):
182     """Return a nicer format for build time
183
184     >>> format_build_time('300')
185     '0:05:00'
186     """
187     return str(datetime.timedelta(seconds=int(duration)))
188
189
190 def default_build_triple(verbose):
191     """Build triple as in LLVM"""
192     # If the user already has a host build triple with an existing `rustc`
193     # install, use their preference. This fixes most issues with Windows builds
194     # being detected as GNU instead of MSVC.
195     default_encoding = sys.getdefaultencoding()
196     try:
197         version = subprocess.check_output(["rustc", "--version", "--verbose"],
198                 stderr=subprocess.DEVNULL)
199         version = version.decode(default_encoding)
200         host = next(x for x in version.split('\n') if x.startswith("host: "))
201         triple = host.split("host: ")[1]
202         if verbose:
203             print("detected default triple {}".format(triple))
204         return triple
205     except Exception as e:
206         if verbose:
207             print("rustup not detected: {}".format(e))
208             print("falling back to auto-detect")
209
210     required = sys.platform != 'win32'
211     ostype = require(["uname", "-s"], exit=required)
212     cputype = require(['uname', '-m'], exit=required)
213
214     # If we do not have `uname`, assume Windows.
215     if ostype is None or cputype is None:
216         return 'x86_64-pc-windows-msvc'
217
218     ostype = ostype.decode(default_encoding)
219     cputype = cputype.decode(default_encoding)
220
221     # The goal here is to come up with the same triple as LLVM would,
222     # at least for the subset of platforms we're willing to target.
223     ostype_mapper = {
224         'Darwin': 'apple-darwin',
225         'DragonFly': 'unknown-dragonfly',
226         'FreeBSD': 'unknown-freebsd',
227         'Haiku': 'unknown-haiku',
228         'NetBSD': 'unknown-netbsd',
229         'OpenBSD': 'unknown-openbsd'
230     }
231
232     # Consider the direct transformation first and then the special cases
233     if ostype in ostype_mapper:
234         ostype = ostype_mapper[ostype]
235     elif ostype == 'Linux':
236         os_from_sp = subprocess.check_output(
237             ['uname', '-o']).strip().decode(default_encoding)
238         if os_from_sp == 'Android':
239             ostype = 'linux-android'
240         else:
241             ostype = 'unknown-linux-gnu'
242     elif ostype == 'SunOS':
243         ostype = 'pc-solaris'
244         # On Solaris, uname -m will return a machine classification instead
245         # of a cpu type, so uname -p is recommended instead.  However, the
246         # output from that option is too generic for our purposes (it will
247         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
248         # must be used instead.
249         cputype = require(['isainfo', '-k']).decode(default_encoding)
250         # sparc cpus have sun as a target vendor
251         if 'sparc' in cputype:
252             ostype = 'sun-solaris'
253     elif ostype.startswith('MINGW'):
254         # msys' `uname` does not print gcc configuration, but prints msys
255         # configuration. so we cannot believe `uname -m`:
256         # msys1 is always i686 and msys2 is always x86_64.
257         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
258         # MINGW64 on x86_64.
259         ostype = 'pc-windows-gnu'
260         cputype = 'i686'
261         if os.environ.get('MSYSTEM') == 'MINGW64':
262             cputype = 'x86_64'
263     elif ostype.startswith('MSYS'):
264         ostype = 'pc-windows-gnu'
265     elif ostype.startswith('CYGWIN_NT'):
266         cputype = 'i686'
267         if ostype.endswith('WOW64'):
268             cputype = 'x86_64'
269         ostype = 'pc-windows-gnu'
270     elif sys.platform == 'win32':
271         # Some Windows platforms might have a `uname` command that returns a
272         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
273         # these cases, fall back to using sys.platform.
274         return 'x86_64-pc-windows-msvc'
275     else:
276         err = "unknown OS type: {}".format(ostype)
277         sys.exit(err)
278
279     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
280         cputype = subprocess.check_output(
281               ['uname', '-p']).strip().decode(default_encoding)
282     cputype_mapper = {
283         'BePC': 'i686',
284         'aarch64': 'aarch64',
285         'amd64': 'x86_64',
286         'arm64': 'aarch64',
287         'i386': 'i686',
288         'i486': 'i686',
289         'i686': 'i686',
290         'i786': 'i686',
291         'powerpc': 'powerpc',
292         'powerpc64': 'powerpc64',
293         'powerpc64le': 'powerpc64le',
294         'ppc': 'powerpc',
295         'ppc64': 'powerpc64',
296         'ppc64le': 'powerpc64le',
297         's390x': 's390x',
298         'x64': 'x86_64',
299         'x86': 'i686',
300         'x86-64': 'x86_64',
301         'x86_64': 'x86_64'
302     }
303
304     # Consider the direct transformation first and then the special cases
305     if cputype in cputype_mapper:
306         cputype = cputype_mapper[cputype]
307     elif cputype in {'xscale', 'arm'}:
308         cputype = 'arm'
309         if ostype == 'linux-android':
310             ostype = 'linux-androideabi'
311         elif ostype == 'unknown-freebsd':
312             cputype = subprocess.check_output(
313                 ['uname', '-p']).strip().decode(default_encoding)
314             ostype = 'unknown-freebsd'
315     elif cputype == 'armv6l':
316         cputype = 'arm'
317         if ostype == 'linux-android':
318             ostype = 'linux-androideabi'
319         else:
320             ostype += 'eabihf'
321     elif cputype in {'armv7l', 'armv8l'}:
322         cputype = 'armv7'
323         if ostype == 'linux-android':
324             ostype = 'linux-androideabi'
325         else:
326             ostype += 'eabihf'
327     elif cputype == 'mips':
328         if sys.byteorder == 'big':
329             cputype = 'mips'
330         elif sys.byteorder == 'little':
331             cputype = 'mipsel'
332         else:
333             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
334     elif cputype == 'mips64':
335         if sys.byteorder == 'big':
336             cputype = 'mips64'
337         elif sys.byteorder == 'little':
338             cputype = 'mips64el'
339         else:
340             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
341         # only the n64 ABI is supported, indicate it
342         ostype += 'abi64'
343     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
344         pass
345     else:
346         err = "unknown cpu type: {}".format(cputype)
347         sys.exit(err)
348
349     return "{}-{}".format(cputype, ostype)
350
351
352 @contextlib.contextmanager
353 def output(filepath):
354     tmp = filepath + '.tmp'
355     with open(tmp, 'w') as f:
356         yield f
357     try:
358         if os.path.exists(filepath):
359             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
360     except OSError:
361         shutil.copy2(tmp, filepath)
362         os.remove(tmp)
363         return
364     os.rename(tmp, filepath)
365
366
367 class RustBuild(object):
368     """Provide all the methods required to build Rust"""
369     def __init__(self):
370         self.date = ''
371         self._download_url = ''
372         self.rustc_channel = ''
373         self.rustfmt_channel = ''
374         self.build = ''
375         self.build_dir = ''
376         self.clean = False
377         self.config_toml = ''
378         self.rust_root = ''
379         self.use_locked_deps = ''
380         self.use_vendored_sources = ''
381         self.verbose = False
382         self.git_version = None
383         self.nix_deps_dir = None
384         self.rustc_commit = None
385
386     def download_stage0(self):
387         """Fetch the build system for Rust, written in Rust
388
389         This method will build a cache directory, then it will fetch the
390         tarball which has the stage0 compiler used to then bootstrap the Rust
391         compiler itself.
392
393         Each downloaded tarball is extracted, after that, the script
394         will move all the content to the right place.
395         """
396         rustc_channel = self.rustc_channel
397         rustfmt_channel = self.rustfmt_channel
398
399         if self.rustc().startswith(self.bin_root()) and \
400                 (not os.path.exists(self.rustc()) or
401                  self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
402             if os.path.exists(self.bin_root()):
403                 shutil.rmtree(self.bin_root())
404             download_rustc = self.rustc_commit is not None
405             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
406             filename = "rust-std-{}-{}{}".format(
407                 rustc_channel, self.build, tarball_suffix)
408             pattern = "rust-std-{}".format(self.build)
409             self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
410             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
411                                               tarball_suffix)
412             self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
413             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
414                                               tarball_suffix)
415             self._download_component_helper(filename, "cargo", tarball_suffix)
416             if self.rustc_commit is not None:
417                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
418                 self._download_component_helper(
419                     filename, "rustc-dev", tarball_suffix, download_rustc
420                 )
421
422             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
423             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
424             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
425             lib_dir = "{}/lib".format(self.bin_root())
426             for lib in os.listdir(lib_dir):
427                 if lib.endswith(".so"):
428                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
429             with output(self.rustc_stamp()) as rust_stamp:
430                 rust_stamp.write(self.date + str(self.rustc_commit))
431
432         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
433             not os.path.exists(self.rustfmt())
434             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
435         ):
436             if rustfmt_channel:
437                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
438                 [channel, date] = rustfmt_channel.split('-', 1)
439                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
440                 self._download_component_helper(
441                     filename, "rustfmt-preview", tarball_suffix, key=date
442                 )
443                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
444                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
445                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
446                     rustfmt_stamp.write(self.rustfmt_channel)
447
448         if self.downloading_llvm():
449             # We want the most recent LLVM submodule update to avoid downloading
450             # LLVM more often than necessary.
451             #
452             # This git command finds that commit SHA, looking for bors-authored
453             # merges that modified src/llvm-project.
454             #
455             # This works even in a repository that has not yet initialized
456             # submodules.
457             top_level = subprocess.check_output([
458                 "git", "rev-parse", "--show-toplevel",
459             ]).decode(sys.getdefaultencoding()).strip()
460             llvm_sha = subprocess.check_output([
461                 "git", "log", "--author=bors", "--format=%H", "-n1",
462                 "-m", "--first-parent",
463                 "--",
464                 "{}/src/llvm-project".format(top_level),
465                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
466             ]).decode(sys.getdefaultencoding()).strip()
467             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
468             llvm_root = self.llvm_root()
469             llvm_lib = os.path.join(llvm_root, "lib")
470             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
471                 self._download_ci_llvm(llvm_sha, llvm_assertions)
472                 for binary in ["llvm-config", "FileCheck"]:
473                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary), rpath_libz=True)
474                 for lib in os.listdir(llvm_lib):
475                     if lib.endswith(".so"):
476                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib), rpath_libz=True)
477                 with output(self.llvm_stamp()) as llvm_stamp:
478                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
479
480     def downloading_llvm(self):
481         opt = self.get_toml('download-ci-llvm', 'llvm')
482         # This is currently all tier 1 targets (since others may not have CI
483         # artifacts)
484         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
485         supported_platforms = [
486             "aarch64-unknown-linux-gnu",
487             "i686-pc-windows-gnu",
488             "i686-pc-windows-msvc",
489             "i686-unknown-linux-gnu",
490             "x86_64-unknown-linux-gnu",
491             "x86_64-apple-darwin",
492             "x86_64-pc-windows-gnu",
493             "x86_64-pc-windows-msvc",
494         ]
495         return opt == "true" \
496             or (opt == "if-available" and self.build in supported_platforms)
497
498     def _download_component_helper(
499         self, filename, pattern, tarball_suffix, download_rustc=False, key=None
500     ):
501         if key is None:
502             if download_rustc:
503                 key = self.rustc_commit
504             else:
505                 key = self.date
506         cache_dst = os.path.join(self.build_dir, "cache")
507         rustc_cache = os.path.join(cache_dst, key)
508         if not os.path.exists(rustc_cache):
509             os.makedirs(rustc_cache)
510
511         if download_rustc:
512             url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
513         else:
514             url = "{}/dist/{}".format(self._download_url, key)
515         tarball = os.path.join(rustc_cache, filename)
516         if not os.path.exists(tarball):
517             do_verify = not download_rustc
518             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
519         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
520
521     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
522         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
523         cache_dst = os.path.join(self.build_dir, "cache")
524         rustc_cache = os.path.join(cache_dst, cache_prefix)
525         if not os.path.exists(rustc_cache):
526             os.makedirs(rustc_cache)
527
528         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
529         if llvm_assertions:
530             url = url.replace('rustc-builds', 'rustc-builds-alt')
531         # ci-artifacts are only stored as .xz, not .gz
532         if not support_xz():
533             print("error: XZ support is required to download LLVM")
534             print("help: consider disabling `download-ci-llvm` or using python3")
535             exit(1)
536         tarball_suffix = '.tar.xz'
537         filename = "rust-dev-nightly-" + self.build + tarball_suffix
538         tarball = os.path.join(rustc_cache, filename)
539         if not os.path.exists(tarball):
540             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
541         unpack(tarball, tarball_suffix, self.llvm_root(),
542                 match="rust-dev",
543                 verbose=self.verbose)
544
545     def fix_bin_or_dylib(self, fname, rpath_libz=False):
546         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
547         or the RPATH section, to fix the dynamic library search path
548
549         This method is only required on NixOS and uses the PatchELF utility to
550         change the interpreter/RPATH of ELF executables.
551
552         Please see https://nixos.org/patchelf.html for more information
553         """
554         default_encoding = sys.getdefaultencoding()
555         try:
556             ostype = subprocess.check_output(
557                 ['uname', '-s']).strip().decode(default_encoding)
558         except subprocess.CalledProcessError:
559             return
560         except OSError as reason:
561             if getattr(reason, 'winerror', None) is not None:
562                 return
563             raise reason
564
565         if ostype != "Linux":
566             return
567
568         if not os.path.exists("/etc/NIXOS"):
569             return
570         if os.path.exists("/lib"):
571             return
572
573         # At this point we're pretty sure the user is running NixOS
574         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
575         print(nix_os_msg, fname)
576
577         # Only build `stage0/.nix-deps` once.
578         nix_deps_dir = self.nix_deps_dir
579         if not nix_deps_dir:
580             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
581             if not os.path.exists(nix_deps_dir):
582                 os.makedirs(nix_deps_dir)
583
584             nix_deps = [
585                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
586                 "stdenv.cc.bintools",
587
588                 # Needed as a system dependency of `libLLVM-*.so`.
589                 "zlib",
590
591                 # Needed for patching ELF binaries (see doc comment above).
592                 "patchelf",
593             ]
594
595             # Run `nix-build` to "build" each dependency (which will likely reuse
596             # the existing `/nix/store` copy, or at most download a pre-built copy).
597             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
598             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
599             # ensuring garbage collection will never remove the `/nix/store` path
600             # (which would break our patched binaries that hardcode those paths).
601             for dep in nix_deps:
602                 try:
603                     subprocess.check_output([
604                         "nix-build", "<nixpkgs>",
605                         "-A", dep,
606                         "-o", "{}/{}".format(nix_deps_dir, dep),
607                     ])
608                 except subprocess.CalledProcessError as reason:
609                     print("warning: failed to call nix-build:", reason)
610                     return
611
612             self.nix_deps_dir = nix_deps_dir
613
614         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
615         patchelf_args = []
616
617         if rpath_libz:
618             # Patch RPATH to add `zlib` dependency that stems from LLVM
619             dylib_deps = ["zlib"]
620             rpath_entries = [
621                 # Relative default, all binary and dynamic libraries we ship
622                 # appear to have this (even when `../lib` is redundant).
623                 "$ORIGIN/../lib",
624             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
625             patchelf_args += ["--set-rpath", ":".join(rpath_entries)]
626         if not fname.endswith(".so"):
627             # Finally, set the corret .interp for binaries
628             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
629             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
630                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
631
632         try:
633             subprocess.check_output([patchelf] + patchelf_args + [fname])
634         except subprocess.CalledProcessError as reason:
635             print("warning: failed to call patchelf:", reason)
636             return
637
638     # Return the stage1 compiler to download, if any.
639     def maybe_download_rustc(self):
640         # If `download-rustc` is not set, default to rebuilding.
641         if self.get_toml("download-rustc", section="rust") != "true":
642             return None
643
644         # Handle running from a directory other than the top level
645         rev_parse = ["git", "rev-parse", "--show-toplevel"]
646         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
647         compiler = "{}/compiler/".format(top_level)
648
649         # Look for a version to compare to based on the current commit.
650         # Only commits merged by bors will have CI artifacts.
651         merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"]
652         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
653
654         # Warn if there were changes to the compiler since the ancestor commit.
655         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
656         if status != 0:
657             print("warning: `download-rustc` is enabled, but there are changes to compiler/")
658
659         return commit
660
661     def rustc_stamp(self):
662         """Return the path for .rustc-stamp
663
664         >>> rb = RustBuild()
665         >>> rb.build_dir = "build"
666         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
667         True
668         """
669         return os.path.join(self.bin_root(), '.rustc-stamp')
670
671     def rustfmt_stamp(self):
672         """Return the path for .rustfmt-stamp
673
674         >>> rb = RustBuild()
675         >>> rb.build_dir = "build"
676         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
677         True
678         """
679         return os.path.join(self.bin_root(), '.rustfmt-stamp')
680
681     def llvm_stamp(self):
682         """Return the path for .rustfmt-stamp
683
684         >>> rb = RustBuild()
685         >>> rb.build_dir = "build"
686         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
687         True
688         """
689         return os.path.join(self.llvm_root(), '.llvm-stamp')
690
691
692     def program_out_of_date(self, stamp_path, key):
693         """Check if the given program stamp is out of date"""
694         if not os.path.exists(stamp_path) or self.clean:
695             return True
696         with open(stamp_path, 'r') as stamp:
697             return key != stamp.read()
698
699     def bin_root(self):
700         """Return the binary root directory
701
702         >>> rb = RustBuild()
703         >>> rb.build_dir = "build"
704         >>> rb.bin_root() == os.path.join("build", "stage0")
705         True
706
707         When the 'build' property is given should be a nested directory:
708
709         >>> rb.build = "devel"
710         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
711         True
712         """
713         return os.path.join(self.build_dir, self.build, "stage0")
714
715     def llvm_root(self):
716         """Return the CI LLVM root directory
717
718         >>> rb = RustBuild()
719         >>> rb.build_dir = "build"
720         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
721         True
722
723         When the 'build' property is given should be a nested directory:
724
725         >>> rb.build = "devel"
726         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
727         True
728         """
729         return os.path.join(self.build_dir, self.build, "ci-llvm")
730
731     def get_toml(self, key, section=None):
732         """Returns the value of the given key in config.toml, otherwise returns None
733
734         >>> rb = RustBuild()
735         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
736         >>> rb.get_toml("key2")
737         'value2'
738
739         If the key does not exists, the result is None:
740
741         >>> rb.get_toml("key3") is None
742         True
743
744         Optionally also matches the section the key appears in
745
746         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
747         >>> rb.get_toml('key', 'a')
748         'value1'
749         >>> rb.get_toml('key', 'b')
750         'value2'
751         >>> rb.get_toml('key', 'c') is None
752         True
753
754         >>> rb.config_toml = 'key1 = true'
755         >>> rb.get_toml("key1")
756         'true'
757         """
758
759         cur_section = None
760         for line in self.config_toml.splitlines():
761             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
762             if section_match is not None:
763                 cur_section = section_match.group(1)
764
765             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
766             if match is not None:
767                 value = match.group(1)
768                 if section is None or section == cur_section:
769                     return self.get_string(value) or value.strip()
770         return None
771
772     def cargo(self):
773         """Return config path for cargo"""
774         return self.program_config('cargo')
775
776     def rustc(self):
777         """Return config path for rustc"""
778         return self.program_config('rustc')
779
780     def rustfmt(self):
781         """Return config path for rustfmt"""
782         if not self.rustfmt_channel:
783             return None
784         return self.program_config('rustfmt')
785
786     def program_config(self, program):
787         """Return config path for the given program
788
789         >>> rb = RustBuild()
790         >>> rb.config_toml = 'rustc = "rustc"\\n'
791         >>> rb.program_config('rustc')
792         'rustc'
793         >>> rb.config_toml = ''
794         >>> cargo_path = rb.program_config('cargo')
795         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
796         ... "bin", "cargo")
797         True
798         """
799         config = self.get_toml(program)
800         if config:
801             return os.path.expanduser(config)
802         return os.path.join(self.bin_root(), "bin", "{}{}".format(
803             program, self.exe_suffix()))
804
805     @staticmethod
806     def get_string(line):
807         """Return the value between double quotes
808
809         >>> RustBuild.get_string('    "devel"   ')
810         'devel'
811         >>> RustBuild.get_string("    'devel'   ")
812         'devel'
813         >>> RustBuild.get_string('devel') is None
814         True
815         >>> RustBuild.get_string('    "devel   ')
816         ''
817         """
818         start = line.find('"')
819         if start != -1:
820             end = start + 1 + line[start + 1:].find('"')
821             return line[start + 1:end]
822         start = line.find('\'')
823         if start != -1:
824             end = start + 1 + line[start + 1:].find('\'')
825             return line[start + 1:end]
826         return None
827
828     @staticmethod
829     def exe_suffix():
830         """Return a suffix for executables"""
831         if sys.platform == 'win32':
832             return '.exe'
833         return ''
834
835     def bootstrap_binary(self):
836         """Return the path of the bootstrap binary
837
838         >>> rb = RustBuild()
839         >>> rb.build_dir = "build"
840         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
841         ... "debug", "bootstrap")
842         True
843         """
844         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
845
846     def build_bootstrap(self):
847         """Build bootstrap"""
848         build_dir = os.path.join(self.build_dir, "bootstrap")
849         if self.clean and os.path.exists(build_dir):
850             shutil.rmtree(build_dir)
851         env = os.environ.copy()
852         # `CARGO_BUILD_TARGET` breaks bootstrap build.
853         # See also: <https://github.com/rust-lang/rust/issues/70208>.
854         if "CARGO_BUILD_TARGET" in env:
855             del env["CARGO_BUILD_TARGET"]
856         env["CARGO_TARGET_DIR"] = build_dir
857         env["RUSTC"] = self.rustc()
858         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
859             (os.pathsep + env["LD_LIBRARY_PATH"]) \
860             if "LD_LIBRARY_PATH" in env else ""
861         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
862             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
863             if "DYLD_LIBRARY_PATH" in env else ""
864         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
865             (os.pathsep + env["LIBRARY_PATH"]) \
866             if "LIBRARY_PATH" in env else ""
867         # preserve existing RUSTFLAGS
868         env.setdefault("RUSTFLAGS", "")
869         env["RUSTFLAGS"] += " -Cdebuginfo=2"
870
871         build_section = "target.{}".format(self.build)
872         target_features = []
873         if self.get_toml("crt-static", build_section) == "true":
874             target_features += ["+crt-static"]
875         elif self.get_toml("crt-static", build_section) == "false":
876             target_features += ["-crt-static"]
877         if target_features:
878             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
879         target_linker = self.get_toml("linker", build_section)
880         if target_linker is not None:
881             env["RUSTFLAGS"] += " -C linker=" + target_linker
882         # cfg(bootstrap): Add `-Wsemicolon_in_expressions_from_macros` after the next beta bump
883         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
884         if self.get_toml("deny-warnings", "rust") != "false":
885             env["RUSTFLAGS"] += " -Dwarnings"
886
887         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
888             os.pathsep + env["PATH"]
889         if not os.path.isfile(self.cargo()):
890             raise Exception("no cargo executable found at `{}`".format(
891                 self.cargo()))
892         args = [self.cargo(), "build", "--manifest-path",
893                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
894         for _ in range(1, self.verbose):
895             args.append("--verbose")
896         if self.use_locked_deps:
897             args.append("--locked")
898         if self.use_vendored_sources:
899             args.append("--frozen")
900         run(args, env=env, verbose=self.verbose)
901
902     def build_triple(self):
903         """Build triple as in LLVM
904
905         Note that `default_build_triple` is moderately expensive,
906         so use `self.build` where possible.
907         """
908         config = self.get_toml('build')
909         if config:
910             return config
911         return default_build_triple(self.verbose)
912
913     def check_submodule(self, module, slow_submodules):
914         if not slow_submodules:
915             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
916                                            cwd=os.path.join(self.rust_root, module),
917                                            stdout=subprocess.PIPE)
918             return checked_out
919         else:
920             return None
921
922     def update_submodule(self, module, checked_out, recorded_submodules):
923         module_path = os.path.join(self.rust_root, module)
924
925         if checked_out is not None:
926             default_encoding = sys.getdefaultencoding()
927             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
928             if recorded_submodules[module] == checked_out:
929                 return
930
931         print("Updating submodule", module)
932
933         run(["git", "submodule", "-q", "sync", module],
934             cwd=self.rust_root, verbose=self.verbose)
935
936         update_args = ["git", "submodule", "update", "--init", "--recursive"]
937         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
938             update_args.append("--progress")
939         update_args.append(module)
940         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
941
942         run(["git", "reset", "-q", "--hard"],
943             cwd=module_path, verbose=self.verbose)
944         run(["git", "clean", "-qdfx"],
945             cwd=module_path, verbose=self.verbose)
946
947     def update_submodules(self):
948         """Update submodules"""
949         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
950                 self.get_toml('submodules') == "false":
951             return
952
953         default_encoding = sys.getdefaultencoding()
954
955         # check the existence and version of 'git' command
956         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
957         self.git_version = distutils.version.LooseVersion(git_version_str)
958
959         slow_submodules = self.get_toml('fast-submodules') == "false"
960         start_time = time()
961         if slow_submodules:
962             print('Unconditionally updating all submodules')
963         else:
964             print('Updating only changed submodules')
965         default_encoding = sys.getdefaultencoding()
966         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
967             ["git", "config", "--file",
968              os.path.join(self.rust_root, ".gitmodules"),
969              "--get-regexp", "path"]
970         ).decode(default_encoding).splitlines()]
971         filtered_submodules = []
972         submodules_names = []
973         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
974         external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
975         llvm_needed = not self.get_toml('codegen-backends', 'rust') \
976             or "llvm" in self.get_toml('codegen-backends', 'rust')
977         for module in submodules:
978             if module.endswith("llvm-project"):
979                 # Don't sync the llvm-project submodule if an external LLVM was
980                 # provided, if we are downloading LLVM or if the LLVM backend is
981                 # not being built. Also, if the submodule has been initialized
982                 # already, sync it anyways so that it doesn't mess up contributor
983                 # pull requests.
984                 if external_llvm_provided or not llvm_needed:
985                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
986                         continue
987             check = self.check_submodule(module, slow_submodules)
988             filtered_submodules.append((module, check))
989             submodules_names.append(module)
990         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
991                                     cwd=self.rust_root, stdout=subprocess.PIPE)
992         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
993         recorded_submodules = {}
994         for data in recorded:
995             data = data.split()
996             recorded_submodules[data[3]] = data[2]
997         for module in filtered_submodules:
998             self.update_submodule(module[0], module[1], recorded_submodules)
999         print("Submodules updated in %.2f seconds" % (time() - start_time))
1000
1001     def set_normal_environment(self):
1002         """Set download URL for normal environment"""
1003         if 'RUSTUP_DIST_SERVER' in os.environ:
1004             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1005         else:
1006             self._download_url = 'https://static.rust-lang.org'
1007
1008     def set_dev_environment(self):
1009         """Set download URL for development environment"""
1010         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1011             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1012         else:
1013             self._download_url = 'https://dev-static.rust-lang.org'
1014
1015     def check_vendored_status(self):
1016         """Check that vendoring is configured properly"""
1017         vendor_dir = os.path.join(self.rust_root, 'vendor')
1018         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1019             if os.environ.get('USER') != os.environ['SUDO_USER']:
1020                 self.use_vendored_sources = True
1021                 print('info: looks like you are running this command under `sudo`')
1022                 print('      and so in order to preserve your $HOME this will now')
1023                 print('      use vendored sources by default.')
1024                 if not os.path.exists(vendor_dir):
1025                     print('error: vendoring required, but vendor directory does not exist.')
1026                     print('       Run `cargo vendor` without sudo to initialize the '
1027                           'vendor directory.')
1028                     raise Exception("{} not found".format(vendor_dir))
1029
1030         if self.use_vendored_sources:
1031             if not os.path.exists('.cargo'):
1032                 os.makedirs('.cargo')
1033             with output('.cargo/config') as cargo_config:
1034                 cargo_config.write(
1035                     "[source.crates-io]\n"
1036                     "replace-with = 'vendored-sources'\n"
1037                     "registry = 'https://example.com'\n"
1038                     "\n"
1039                     "[source.vendored-sources]\n"
1040                     "directory = '{}/vendor'\n"
1041                     .format(self.rust_root))
1042         else:
1043             if os.path.exists('.cargo'):
1044                 shutil.rmtree('.cargo')
1045
1046     def ensure_vendored(self):
1047         """Ensure that the vendored sources are available if needed"""
1048         vendor_dir = os.path.join(self.rust_root, 'vendor')
1049         # Note that this does not handle updating the vendored dependencies if
1050         # the rust git repository is updated. Normal development usually does
1051         # not use vendoring, so hopefully this isn't too much of a problem.
1052         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1053             run([
1054                 self.cargo(),
1055                 "vendor",
1056                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1057                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1058             ], verbose=self.verbose, cwd=self.rust_root)
1059
1060
1061 def bootstrap(help_triggered):
1062     """Configure, fetch, build and run the initial bootstrap"""
1063
1064     # If the user is asking for help, let them know that the whole download-and-build
1065     # process has to happen before anything is printed out.
1066     if help_triggered:
1067         print("info: Downloading and building bootstrap before processing --help")
1068         print("      command. See src/bootstrap/README.md for help with common")
1069         print("      commands.")
1070
1071     parser = argparse.ArgumentParser(description='Build rust')
1072     parser.add_argument('--config')
1073     parser.add_argument('--build')
1074     parser.add_argument('--clean', action='store_true')
1075     parser.add_argument('-v', '--verbose', action='count', default=0)
1076
1077     args = [a for a in sys.argv if a != '-h' and a != '--help']
1078     args, _ = parser.parse_known_args(args)
1079
1080     # Configure initial bootstrap
1081     build = RustBuild()
1082     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1083     build.verbose = args.verbose
1084     build.clean = args.clean
1085
1086     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1087     # exists).
1088     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1089     if not toml_path and os.path.exists('config.toml'):
1090         toml_path = 'config.toml'
1091
1092     if toml_path:
1093         if not os.path.exists(toml_path):
1094             toml_path = os.path.join(build.rust_root, toml_path)
1095
1096         with open(toml_path) as config:
1097             build.config_toml = config.read()
1098
1099     profile = build.get_toml('profile')
1100     if profile is not None:
1101         include_file = 'config.{}.toml'.format(profile)
1102         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1103         include_path = os.path.join(include_dir, include_file)
1104         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1105         # specific key, so appending our defaults at the end allows the user to override them
1106         with open(include_path) as included_toml:
1107             build.config_toml += os.linesep + included_toml.read()
1108
1109     config_verbose = build.get_toml('verbose', 'build')
1110     if config_verbose is not None:
1111         build.verbose = max(build.verbose, int(config_verbose))
1112
1113     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1114
1115     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1116
1117     build.check_vendored_status()
1118
1119     build_dir = build.get_toml('build-dir', 'build') or 'build'
1120     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1121
1122     data = stage0_data(build.rust_root)
1123     build.date = data['date']
1124     build.rustc_channel = data['rustc']
1125
1126     if "rustfmt" in data:
1127         build.rustfmt_channel = data['rustfmt']
1128
1129     if 'dev' in data:
1130         build.set_dev_environment()
1131     else:
1132         build.set_normal_environment()
1133
1134     build.build = args.build or build.build_triple()
1135     build.update_submodules()
1136
1137     # Fetch/build the bootstrap
1138     build.rustc_commit = build.maybe_download_rustc()
1139     if build.rustc_commit is not None:
1140         if build.verbose:
1141             commit = build.rustc_commit
1142             print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
1143         # FIXME: support downloading artifacts from the beta channel
1144         build.rustc_channel = "nightly"
1145     build.download_stage0()
1146     sys.stdout.flush()
1147     build.ensure_vendored()
1148     build.build_bootstrap()
1149     sys.stdout.flush()
1150
1151     # Run the bootstrap
1152     args = [build.bootstrap_binary()]
1153     args.extend(sys.argv[1:])
1154     env = os.environ.copy()
1155     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1156     env["BOOTSTRAP_PYTHON"] = sys.executable
1157     env["BUILD_DIR"] = build.build_dir
1158     env["RUSTC_BOOTSTRAP"] = '1'
1159     if toml_path:
1160         env["BOOTSTRAP_CONFIG"] = toml_path
1161     run(args, env=env, verbose=build.verbose)
1162
1163
1164 def main():
1165     """Entry point for the bootstrap process"""
1166     start_time = time()
1167
1168     # x.py help <cmd> ...
1169     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1170         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1171
1172     help_triggered = (
1173         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1174     try:
1175         bootstrap(help_triggered)
1176         if not help_triggered:
1177             print("Build completed successfully in {}".format(
1178                 format_build_time(time() - start_time)))
1179     except (SystemExit, KeyboardInterrupt) as error:
1180         if hasattr(error, 'code') and isinstance(error.code, int):
1181             exit_code = error.code
1182         else:
1183             exit_code = 1
1184             print(error)
1185         if not help_triggered:
1186             print("Build completed unsuccessfully in {}".format(
1187                 format_build_time(time() - start_time)))
1188         sys.exit(exit_code)
1189
1190
1191 if __name__ == '__main__':
1192     main()