]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
Number of filtered out tests in tests summary
[rust.git] / src / bootstrap / bootstrap.py
1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
4 #
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
10
11 from __future__ import print_function
12 import argparse
13 import contextlib
14 import datetime
15 import hashlib
16 import os
17 import shutil
18 import subprocess
19 import sys
20 import tarfile
21 import tempfile
22
23 from time import time
24
25
26 def get(url, path, verbose=False):
27     sha_url = url + ".sha256"
28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
29         temp_path = temp_file.name
30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
31         sha_path = sha_file.name
32
33     try:
34         download(sha_path, sha_url, False, verbose)
35         if os.path.exists(path):
36             if verify(path, sha_path, False):
37                 if verbose:
38                     print("using already-download file " + path)
39                 return
40             else:
41                 if verbose:
42                     print("ignoring already-download file " + path + " due to failed verification")
43                 os.unlink(path)
44         download(temp_path, url, True, verbose)
45         if not verify(temp_path, sha_path, verbose):
46             raise RuntimeError("failed verification")
47         if verbose:
48             print("moving {} to {}".format(temp_path, path))
49         shutil.move(temp_path, path)
50     finally:
51         delete_if_present(sha_path, verbose)
52         delete_if_present(temp_path, verbose)
53
54
55 def delete_if_present(path, verbose):
56     if os.path.isfile(path):
57         if verbose:
58             print("removing " + path)
59         os.unlink(path)
60
61
62 def download(path, url, probably_big, verbose):
63     for x in range(0, 4):
64         try:
65             _download(path, url, probably_big, verbose, True)
66             return
67         except RuntimeError:
68             print("\nspurious failure, trying again")
69     _download(path, url, probably_big, verbose, False)
70
71
72 def _download(path, url, probably_big, verbose, exception):
73     if probably_big or verbose:
74         print("downloading {}".format(url))
75     # see http://serverfault.com/questions/301128/how-to-download
76     if sys.platform == 'win32':
77         run(["PowerShell.exe", "/nologo", "-Command",
78              "(New-Object System.Net.WebClient)"
79              ".DownloadFile('{}', '{}')".format(url, path)],
80             verbose=verbose,
81             exception=exception)
82     else:
83         if probably_big or verbose:
84             option = "-#"
85         else:
86             option = "-s"
87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
88             verbose=verbose,
89             exception=exception)
90
91
92 def verify(path, sha_path, verbose):
93     if verbose:
94         print("verifying " + path)
95     with open(path, "rb") as f:
96         found = hashlib.sha256(f.read()).hexdigest()
97     with open(sha_path, "r") as f:
98         expected = f.readline().split()[0]
99     verified = found == expected
100     if not verified:
101         print("invalid checksum:\n"
102                "    found:    {}\n"
103                "    expected: {}".format(found, expected))
104     return verified
105
106
107 def unpack(tarball, dst, verbose=False, match=None):
108     print("extracting " + tarball)
109     fname = os.path.basename(tarball).replace(".tar.gz", "")
110     with contextlib.closing(tarfile.open(tarball)) as tar:
111         for p in tar.getnames():
112             if "/" not in p:
113                 continue
114             name = p.replace(fname + "/", "", 1)
115             if match is not None and not name.startswith(match):
116                 continue
117             name = name[len(match) + 1:]
118
119             fp = os.path.join(dst, name)
120             if verbose:
121                 print("  extracting " + p)
122             tar.extract(p, dst)
123             tp = os.path.join(dst, p)
124             if os.path.isdir(tp) and os.path.exists(fp):
125                 continue
126             shutil.move(tp, fp)
127     shutil.rmtree(os.path.join(dst, fname))
128
129 def run(args, verbose=False, exception=False):
130     if verbose:
131         print("running: " + ' '.join(args))
132     sys.stdout.flush()
133     # Use Popen here instead of call() as it apparently allows powershell on
134     # Windows to not lock up waiting for input presumably.
135     ret = subprocess.Popen(args)
136     code = ret.wait()
137     if code != 0:
138         err = "failed to run: " + ' '.join(args)
139         if verbose or exception:
140             raise RuntimeError(err)
141         sys.exit(err)
142
143 def stage0_data(rust_root):
144     nightlies = os.path.join(rust_root, "src/stage0.txt")
145     data = {}
146     with open(nightlies, 'r') as nightlies:
147         for line in nightlies:
148             line = line.rstrip()  # Strip newline character, '\n'
149             if line.startswith("#") or line == '':
150                 continue
151             a, b = line.split(": ", 1)
152             data[a] = b
153     return data
154
155 def format_build_time(duration):
156     return str(datetime.timedelta(seconds=int(duration)))
157
158
159 class RustBuild(object):
160     def download_stage0(self):
161         cache_dst = os.path.join(self.build_dir, "cache")
162         rustc_cache = os.path.join(cache_dst, self.stage0_date())
163         if not os.path.exists(rustc_cache):
164             os.makedirs(rustc_cache)
165
166         rustc_channel = self.stage0_rustc_channel()
167         cargo_channel = self.stage0_cargo_channel()
168
169         if self.rustc().startswith(self.bin_root()) and \
170                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
171             self.print_what_it_means_to_bootstrap()
172             if os.path.exists(self.bin_root()):
173                 shutil.rmtree(self.bin_root())
174             filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build)
175             url = self._download_url + "/dist/" + self.stage0_date()
176             tarball = os.path.join(rustc_cache, filename)
177             if not os.path.exists(tarball):
178                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
179             unpack(tarball, self.bin_root(),
180                    match="rust-std-" + self.build,
181                    verbose=self.verbose)
182
183             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
184             url = self._download_url + "/dist/" + self.stage0_date()
185             tarball = os.path.join(rustc_cache, filename)
186             if not os.path.exists(tarball):
187                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
188             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
189             self.fix_executable(self.bin_root() + "/bin/rustc")
190             self.fix_executable(self.bin_root() + "/bin/rustdoc")
191             with open(self.rustc_stamp(), 'w') as f:
192                 f.write(self.stage0_date())
193
194             if "pc-windows-gnu" in self.build:
195                 filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build)
196                 url = self._download_url + "/dist/" + self.stage0_date()
197                 tarball = os.path.join(rustc_cache, filename)
198                 if not os.path.exists(tarball):
199                     get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
200                 unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
201
202         if self.cargo().startswith(self.bin_root()) and \
203                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
204             self.print_what_it_means_to_bootstrap()
205             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
206             url = self._download_url + "/dist/" + self.stage0_date()
207             tarball = os.path.join(rustc_cache, filename)
208             if not os.path.exists(tarball):
209                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
210             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
211             self.fix_executable(self.bin_root() + "/bin/cargo")
212             with open(self.cargo_stamp(), 'w') as f:
213                 f.write(self.stage0_date())
214
215     def fix_executable(self, fname):
216         # If we're on NixOS we need to change the path to the dynamic loader
217
218         default_encoding = sys.getdefaultencoding()
219         try:
220             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
221         except (subprocess.CalledProcessError, WindowsError):
222             return
223
224         if ostype != "Linux":
225             return
226
227         if not os.path.exists("/etc/NIXOS"):
228             return
229         if os.path.exists("/lib"):
230             return
231
232         # At this point we're pretty sure the user is running NixOS
233         print("info: you seem to be running NixOS. Attempting to patch " + fname)
234
235         try:
236             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
237             interpreter = interpreter.strip().decode(default_encoding)
238         except subprocess.CalledProcessError as e:
239             print("warning: failed to call patchelf: %s" % e)
240             return
241
242         loader = interpreter.split("/")[-1]
243
244         try:
245             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
246             ldd_output = ldd_output.strip().decode(default_encoding)
247         except subprocess.CalledProcessError as e:
248             print("warning: unable to call ldd: %s" % e)
249             return
250
251         for line in ldd_output.splitlines():
252             libname = line.split()[0]
253             if libname.endswith(loader):
254                 loader_path = libname[:len(libname) - len(loader)]
255                 break
256         else:
257             print("warning: unable to find the path to the dynamic linker")
258             return
259
260         correct_interpreter = loader_path + loader
261
262         try:
263             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
264         except subprocess.CalledProcessError as e:
265             print("warning: failed to call patchelf: %s" % e)
266             return
267
268     def stage0_date(self):
269         return self._date
270
271     def stage0_rustc_channel(self):
272         return self._rustc_channel
273
274     def stage0_cargo_channel(self):
275         return self._cargo_channel
276
277     def rustc_stamp(self):
278         return os.path.join(self.bin_root(), '.rustc-stamp')
279
280     def cargo_stamp(self):
281         return os.path.join(self.bin_root(), '.cargo-stamp')
282
283     def rustc_out_of_date(self):
284         if not os.path.exists(self.rustc_stamp()) or self.clean:
285             return True
286         with open(self.rustc_stamp(), 'r') as f:
287             return self.stage0_date() != f.read()
288
289     def cargo_out_of_date(self):
290         if not os.path.exists(self.cargo_stamp()) or self.clean:
291             return True
292         with open(self.cargo_stamp(), 'r') as f:
293             return self.stage0_date() != f.read()
294
295     def bin_root(self):
296         return os.path.join(self.build_dir, self.build, "stage0")
297
298     def get_toml(self, key):
299         for line in self.config_toml.splitlines():
300             if line.startswith(key + ' ='):
301                 return self.get_string(line)
302         return None
303
304     def get_mk(self, key):
305         for line in iter(self.config_mk.splitlines()):
306             if line.startswith(key + ' '):
307                 var = line[line.find(':=') + 2:].strip()
308                 if var != '':
309                     return var
310         return None
311
312     def cargo(self):
313         config = self.get_toml('cargo')
314         if config:
315             return config
316         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
317         if config:
318             return config + '/bin/cargo' + self.exe_suffix()
319         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
320
321     def rustc(self):
322         config = self.get_toml('rustc')
323         if config:
324             return config
325         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
326         if config:
327             return config + '/bin/rustc' + self.exe_suffix()
328         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
329
330     def get_string(self, line):
331         start = line.find('"')
332         end = start + 1 + line[start + 1:].find('"')
333         return line[start + 1:end]
334
335     def exe_suffix(self):
336         if sys.platform == 'win32':
337             return '.exe'
338         else:
339             return ''
340
341     def print_what_it_means_to_bootstrap(self):
342         if hasattr(self, 'printed'):
343             return
344         self.printed = True
345         if os.path.exists(self.bootstrap_binary()):
346             return
347         if not '--help' in sys.argv or len(sys.argv) == 1:
348             return
349
350         print('info: the build system for Rust is written in Rust, so this')
351         print('      script is now going to download a stage0 rust compiler')
352         print('      and then compile the build system itself')
353         print('')
354         print('info: in the meantime you can read more about rustbuild at')
355         print('      src/bootstrap/README.md before the download finishes')
356
357     def bootstrap_binary(self):
358         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
359
360     def build_bootstrap(self):
361         self.print_what_it_means_to_bootstrap()
362         build_dir = os.path.join(self.build_dir, "bootstrap")
363         if self.clean and os.path.exists(build_dir):
364             shutil.rmtree(build_dir)
365         env = os.environ.copy()
366         env["CARGO_TARGET_DIR"] = build_dir
367         env["RUSTC"] = self.rustc()
368         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
369                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
370                                  if "LD_LIBRARY_PATH" in env else ""
371         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
372                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
373                                    if "DYLD_LIBRARY_PATH" in env else ""
374         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
375                                    (os.pathsep + env["LIBRARY_PATH"]) \
376                                    if "LIBRARY_PATH" in env else ""
377         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
378                       os.pathsep + env["PATH"]
379         if not os.path.isfile(self.cargo()):
380             raise Exception("no cargo executable found at `%s`" % self.cargo())
381         args = [self.cargo(), "build", "--manifest-path",
382                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
383         if self.use_locked_deps:
384             args.append("--locked")
385         if self.use_vendored_sources:
386             args.append("--frozen")
387         self.run(args, env)
388
389     def run(self, args, env):
390         proc = subprocess.Popen(args, env=env)
391         ret = proc.wait()
392         if ret != 0:
393             sys.exit(ret)
394
395     def build_triple(self):
396         default_encoding = sys.getdefaultencoding()
397         config = self.get_toml('build')
398         if config:
399             return config
400         config = self.get_mk('CFG_BUILD')
401         if config:
402             return config
403         try:
404             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
405             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
406         except (subprocess.CalledProcessError, OSError):
407             if sys.platform == 'win32':
408                 return 'x86_64-pc-windows-msvc'
409             err = "uname not found"
410             if self.verbose:
411                 raise Exception(err)
412             sys.exit(err)
413
414         # The goal here is to come up with the same triple as LLVM would,
415         # at least for the subset of platforms we're willing to target.
416         if ostype == 'Linux':
417             os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
418             if os_from_sp == 'Android':
419                 ostype = 'linux-android'
420             else:
421                 ostype = 'unknown-linux-gnu'
422         elif ostype == 'FreeBSD':
423             ostype = 'unknown-freebsd'
424         elif ostype == 'DragonFly':
425             ostype = 'unknown-dragonfly'
426         elif ostype == 'Bitrig':
427             ostype = 'unknown-bitrig'
428         elif ostype == 'OpenBSD':
429             ostype = 'unknown-openbsd'
430         elif ostype == 'NetBSD':
431             ostype = 'unknown-netbsd'
432         elif ostype == 'SunOS':
433             ostype = 'sun-solaris'
434             # On Solaris, uname -m will return a machine classification instead
435             # of a cpu type, so uname -p is recommended instead.  However, the
436             # output from that option is too generic for our purposes (it will
437             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
438             # must be used instead.
439             try:
440                 cputype = subprocess.check_output(['isainfo',
441                   '-k']).strip().decode(default_encoding)
442             except (subprocess.CalledProcessError, OSError):
443                 err = "isainfo not found"
444                 if self.verbose:
445                     raise Exception(err)
446                 sys.exit(err)
447         elif ostype == 'Darwin':
448             ostype = 'apple-darwin'
449         elif ostype == 'Haiku':
450             ostype = 'unknown-haiku'
451         elif ostype.startswith('MINGW'):
452             # msys' `uname` does not print gcc configuration, but prints msys
453             # configuration. so we cannot believe `uname -m`:
454             # msys1 is always i686 and msys2 is always x86_64.
455             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
456             # MINGW64 on x86_64.
457             ostype = 'pc-windows-gnu'
458             cputype = 'i686'
459             if os.environ.get('MSYSTEM') == 'MINGW64':
460                 cputype = 'x86_64'
461         elif ostype.startswith('MSYS'):
462             ostype = 'pc-windows-gnu'
463         elif ostype.startswith('CYGWIN_NT'):
464             cputype = 'i686'
465             if ostype.endswith('WOW64'):
466                 cputype = 'x86_64'
467             ostype = 'pc-windows-gnu'
468         else:
469             err = "unknown OS type: " + ostype
470             if self.verbose:
471                 raise ValueError(err)
472             sys.exit(err)
473
474         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
475             cputype = 'i686'
476         elif cputype in {'xscale', 'arm'}:
477             cputype = 'arm'
478             if ostype == 'linux-android':
479                 ostype = 'linux-androideabi'
480         elif cputype == 'armv6l':
481             cputype = 'arm'
482             if ostype == 'linux-android':
483                 ostype = 'linux-androideabi'
484             else:
485                 ostype += 'eabihf'
486         elif cputype in {'armv7l', 'armv8l'}:
487             cputype = 'armv7'
488             if ostype == 'linux-android':
489                 ostype = 'linux-androideabi'
490             else:
491                 ostype += 'eabihf'
492         elif cputype in {'aarch64', 'arm64'}:
493             cputype = 'aarch64'
494         elif cputype == 'mips':
495             if sys.byteorder == 'big':
496                 cputype = 'mips'
497             elif sys.byteorder == 'little':
498                 cputype = 'mipsel'
499             else:
500                 raise ValueError('unknown byteorder: ' + sys.byteorder)
501         elif cputype == 'mips64':
502             if sys.byteorder == 'big':
503                 cputype = 'mips64'
504             elif sys.byteorder == 'little':
505                 cputype = 'mips64el'
506             else:
507                 raise ValueError('unknown byteorder: ' + sys.byteorder)
508             # only the n64 ABI is supported, indicate it
509             ostype += 'abi64'
510         elif cputype in {'powerpc', 'ppc'}:
511             cputype = 'powerpc'
512         elif cputype in {'powerpc64', 'ppc64'}:
513             cputype = 'powerpc64'
514         elif cputype in {'powerpc64le', 'ppc64le'}:
515             cputype = 'powerpc64le'
516         elif cputype == 'sparcv9':
517             pass
518         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
519             cputype = 'x86_64'
520         elif cputype == 's390x':
521             cputype = 's390x'
522         elif cputype == 'BePC':
523             cputype = 'i686'
524         else:
525             err = "unknown cpu type: " + cputype
526             if self.verbose:
527                 raise ValueError(err)
528             sys.exit(err)
529
530         return "{}-{}".format(cputype, ostype)
531
532 def bootstrap():
533     parser = argparse.ArgumentParser(description='Build rust')
534     parser.add_argument('--config')
535     parser.add_argument('--clean', action='store_true')
536     parser.add_argument('-v', '--verbose', action='store_true')
537
538     args = [a for a in sys.argv if a != '-h' and a != '--help']
539     args, _ = parser.parse_known_args(args)
540
541     # Configure initial bootstrap
542     rb = RustBuild()
543     rb.config_toml = ''
544     rb.config_mk = ''
545     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
546     rb.build_dir = os.path.join(os.getcwd(), "build")
547     rb.verbose = args.verbose
548     rb.clean = args.clean
549
550     try:
551         with open(args.config or 'config.toml') as config:
552             rb.config_toml = config.read()
553     except:
554         pass
555     try:
556         rb.config_mk = open('config.mk').read()
557     except:
558         pass
559
560     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
561                               'CFG_ENABLE_VENDOR' in rb.config_mk
562
563     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
564                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
565
566     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
567         if os.environ.get('USER') != os.environ['SUDO_USER']:
568             rb.use_vendored_sources = True
569             print('info: looks like you are running this command under `sudo`')
570             print('      and so in order to preserve your $HOME this will now')
571             print('      use vendored sources by default. Note that if this')
572             print('      does not work you should run a normal build first')
573             print('      before running a command like `sudo make install`')
574
575     if rb.use_vendored_sources:
576         if not os.path.exists('.cargo'):
577             os.makedirs('.cargo')
578         with open('.cargo/config','w') as f:
579             f.write("""
580                 [source.crates-io]
581                 replace-with = 'vendored-sources'
582                 registry = 'https://example.com'
583
584                 [source.vendored-sources]
585                 directory = '{}/src/vendor'
586             """.format(rb.rust_root))
587     else:
588         if os.path.exists('.cargo'):
589             shutil.rmtree('.cargo')
590
591     data = stage0_data(rb.rust_root)
592     rb._date = data['date']
593     rb._rustc_channel = data['rustc']
594     rb._cargo_channel = data['cargo']
595     if 'dev' in data:
596         rb._download_url = 'https://dev-static.rust-lang.org'
597     else:
598         rb._download_url = 'https://static.rust-lang.org'
599
600     # Fetch/build the bootstrap
601     rb.build = rb.build_triple()
602     rb.download_stage0()
603     sys.stdout.flush()
604     rb.build_bootstrap()
605     sys.stdout.flush()
606
607     # Run the bootstrap
608     args = [rb.bootstrap_binary()]
609     args.extend(sys.argv[1:])
610     env = os.environ.copy()
611     env["BUILD"] = rb.build
612     env["SRC"] = rb.rust_root
613     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
614     rb.run(args, env)
615
616 def main():
617     start_time = time()
618     help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
619     try:
620         bootstrap()
621         if not help_triggered:
622             print("Build completed successfully in %s" % format_build_time(time() - start_time))
623     except (SystemExit, KeyboardInterrupt) as e:
624         if hasattr(e, 'code') and isinstance(e.code, int):
625             exit_code = e.code
626         else:
627             exit_code = 1
628             print(e)
629         if not help_triggered:
630             print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
631         sys.exit(exit_code)
632
633 if __name__ == '__main__':
634     main()