]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
1dfc5593e5f5eca41444320bb9ea83096428621f
[rust.git] / src / bootstrap / bootstrap.py
1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
4 #
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
10
11 from __future__ import print_function
12 import argparse
13 import contextlib
14 import datetime
15 import hashlib
16 import os
17 import shutil
18 import subprocess
19 import sys
20 import tarfile
21 import tempfile
22
23 from time import time
24
25
26 def get(url, path, verbose=False):
27     sha_url = url + ".sha256"
28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
29         temp_path = temp_file.name
30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
31         sha_path = sha_file.name
32
33     try:
34         download(sha_path, sha_url, False, verbose)
35         if os.path.exists(path):
36             if verify(path, sha_path, False):
37                 if verbose:
38                     print("using already-download file " + path)
39                 return
40             else:
41                 if verbose:
42                     print("ignoring already-download file " + path + " due to failed verification")
43                 os.unlink(path)
44         download(temp_path, url, True, verbose)
45         if not verify(temp_path, sha_path, verbose):
46             raise RuntimeError("failed verification")
47         if verbose:
48             print("moving {} to {}".format(temp_path, path))
49         shutil.move(temp_path, path)
50     finally:
51         delete_if_present(sha_path, verbose)
52         delete_if_present(temp_path, verbose)
53
54
55 def delete_if_present(path, verbose):
56     if os.path.isfile(path):
57         if verbose:
58             print("removing " + path)
59         os.unlink(path)
60
61
62 def download(path, url, probably_big, verbose):
63     for x in range(0, 4):
64         try:
65             _download(path, url, probably_big, verbose, True)
66             return
67         except RuntimeError:
68             print("\nspurious failure, trying again")
69     _download(path, url, probably_big, verbose, False)
70
71
72 def _download(path, url, probably_big, verbose, exception):
73     if probably_big or verbose:
74         print("downloading {}".format(url))
75     # see http://serverfault.com/questions/301128/how-to-download
76     if sys.platform == 'win32':
77         run(["PowerShell.exe", "/nologo", "-Command",
78              "(New-Object System.Net.WebClient)"
79              ".DownloadFile('{}', '{}')".format(url, path)],
80             verbose=verbose,
81             exception=exception)
82     else:
83         if probably_big or verbose:
84             option = "-#"
85         else:
86             option = "-s"
87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
88             verbose=verbose,
89             exception=exception)
90
91
92 def verify(path, sha_path, verbose):
93     if verbose:
94         print("verifying " + path)
95     with open(path, "rb") as f:
96         found = hashlib.sha256(f.read()).hexdigest()
97     with open(sha_path, "r") as f:
98         expected = f.readline().split()[0]
99     verified = found == expected
100     if not verified:
101         print("invalid checksum:\n"
102                "    found:    {}\n"
103                "    expected: {}".format(found, expected))
104     return verified
105
106
107 def unpack(tarball, dst, verbose=False, match=None):
108     print("extracting " + tarball)
109     fname = os.path.basename(tarball).replace(".tar.gz", "")
110     with contextlib.closing(tarfile.open(tarball)) as tar:
111         for p in tar.getnames():
112             if "/" not in p:
113                 continue
114             name = p.replace(fname + "/", "", 1)
115             if match is not None and not name.startswith(match):
116                 continue
117             name = name[len(match) + 1:]
118
119             fp = os.path.join(dst, name)
120             if verbose:
121                 print("  extracting " + p)
122             tar.extract(p, dst)
123             tp = os.path.join(dst, p)
124             if os.path.isdir(tp) and os.path.exists(fp):
125                 continue
126             shutil.move(tp, fp)
127     shutil.rmtree(os.path.join(dst, fname))
128
129 def run(args, verbose=False, exception=False):
130     if verbose:
131         print("running: " + ' '.join(args))
132     sys.stdout.flush()
133     # Use Popen here instead of call() as it apparently allows powershell on
134     # Windows to not lock up waiting for input presumably.
135     ret = subprocess.Popen(args)
136     code = ret.wait()
137     if code != 0:
138         err = "failed to run: " + ' '.join(args)
139         if verbose or exception:
140             raise RuntimeError(err)
141         sys.exit(err)
142
143 def stage0_data(rust_root):
144     nightlies = os.path.join(rust_root, "src/stage0.txt")
145     data = {}
146     with open(nightlies, 'r') as nightlies:
147         for line in nightlies:
148             line = line.rstrip()  # Strip newline character, '\n'
149             if line.startswith("#") or line == '':
150                 continue
151             a, b = line.split(": ", 1)
152             data[a] = b
153     return data
154
155 def format_build_time(duration):
156     return str(datetime.timedelta(seconds=int(duration)))
157
158
159 class RustBuild(object):
160     def download_stage0(self):
161         cache_dst = os.path.join(self.build_dir, "cache")
162         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
163         if not os.path.exists(rustc_cache):
164             os.makedirs(rustc_cache)
165
166         channel = self.stage0_rustc_channel()
167
168         if self.rustc().startswith(self.bin_root()) and \
169                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
170             self.print_what_it_means_to_bootstrap()
171             if os.path.exists(self.bin_root()):
172                 shutil.rmtree(self.bin_root())
173             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
174             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
175             tarball = os.path.join(rustc_cache, filename)
176             if not os.path.exists(tarball):
177                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
178             unpack(tarball, self.bin_root(),
179                    match="rust-std-" + self.build,
180                    verbose=self.verbose)
181
182             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
183             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
184             tarball = os.path.join(rustc_cache, filename)
185             if not os.path.exists(tarball):
186                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
187             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
188             self.fix_executable(self.bin_root() + "/bin/rustc")
189             self.fix_executable(self.bin_root() + "/bin/rustdoc")
190             with open(self.rustc_stamp(), 'w') as f:
191                 f.write(self.stage0_rustc_date())
192
193             if "pc-windows-gnu" in self.build:
194                 filename = "rust-mingw-{}-{}.tar.gz".format(channel, self.build)
195                 url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
196                 tarball = os.path.join(rustc_cache, filename)
197                 if not os.path.exists(tarball):
198                     get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
199                 unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
200
201         if self.cargo().startswith(self.bin_root()) and \
202                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
203             self.print_what_it_means_to_bootstrap()
204             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
205             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
206             tarball = os.path.join(rustc_cache, filename)
207             if not os.path.exists(tarball):
208                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
209             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
210             self.fix_executable(self.bin_root() + "/bin/cargo")
211             with open(self.cargo_stamp(), 'w') as f:
212                 f.write(self.stage0_rustc_date())
213
214     def fix_executable(self, fname):
215         # If we're on NixOS we need to change the path to the dynamic loader
216
217         default_encoding = sys.getdefaultencoding()
218         try:
219             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
220         except (subprocess.CalledProcessError, WindowsError):
221             return
222
223         if ostype != "Linux":
224             return
225
226         if not os.path.exists("/etc/NIXOS"):
227             return
228         if os.path.exists("/lib"):
229             return
230
231         # At this point we're pretty sure the user is running NixOS
232         print("info: you seem to be running NixOS. Attempting to patch " + fname)
233
234         try:
235             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
236             interpreter = interpreter.strip().decode(default_encoding)
237         except subprocess.CalledProcessError as e:
238             print("warning: failed to call patchelf: %s" % e)
239             return
240
241         loader = interpreter.split("/")[-1]
242
243         try:
244             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
245             ldd_output = ldd_output.strip().decode(default_encoding)
246         except subprocess.CalledProcessError as e:
247             print("warning: unable to call ldd: %s" % e)
248             return
249
250         for line in ldd_output.splitlines():
251             libname = line.split()[0]
252             if libname.endswith(loader):
253                 loader_path = libname[:len(libname) - len(loader)]
254                 break
255         else:
256             print("warning: unable to find the path to the dynamic linker")
257             return
258
259         correct_interpreter = loader_path + loader
260
261         try:
262             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
263         except subprocess.CalledProcessError as e:
264             print("warning: failed to call patchelf: %s" % e)
265             return
266
267     def stage0_rustc_date(self):
268         return self._rustc_date
269
270     def stage0_rustc_channel(self):
271         return self._rustc_channel
272
273     def rustc_stamp(self):
274         return os.path.join(self.bin_root(), '.rustc-stamp')
275
276     def cargo_stamp(self):
277         return os.path.join(self.bin_root(), '.cargo-stamp')
278
279     def rustc_out_of_date(self):
280         if not os.path.exists(self.rustc_stamp()) or self.clean:
281             return True
282         with open(self.rustc_stamp(), 'r') as f:
283             return self.stage0_rustc_date() != f.read()
284
285     def cargo_out_of_date(self):
286         if not os.path.exists(self.cargo_stamp()) or self.clean:
287             return True
288         with open(self.cargo_stamp(), 'r') as f:
289             return self.stage0_rustc_date() != f.read()
290
291     def bin_root(self):
292         return os.path.join(self.build_dir, self.build, "stage0")
293
294     def get_toml(self, key):
295         for line in self.config_toml.splitlines():
296             if line.startswith(key + ' ='):
297                 return self.get_string(line)
298         return None
299
300     def get_mk(self, key):
301         for line in iter(self.config_mk.splitlines()):
302             if line.startswith(key + ' '):
303                 var = line[line.find(':=') + 2:].strip()
304                 if var != '':
305                     return var
306         return None
307
308     def cargo(self):
309         config = self.get_toml('cargo')
310         if config:
311             return config
312         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
313         if config:
314             return config + '/bin/cargo' + self.exe_suffix()
315         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
316
317     def rustc(self):
318         config = self.get_toml('rustc')
319         if config:
320             return config
321         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
322         if config:
323             return config + '/bin/rustc' + self.exe_suffix()
324         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
325
326     def get_string(self, line):
327         start = line.find('"')
328         end = start + 1 + line[start + 1:].find('"')
329         return line[start + 1:end]
330
331     def exe_suffix(self):
332         if sys.platform == 'win32':
333             return '.exe'
334         else:
335             return ''
336
337     def print_what_it_means_to_bootstrap(self):
338         if hasattr(self, 'printed'):
339             return
340         self.printed = True
341         if os.path.exists(self.bootstrap_binary()):
342             return
343         if not '--help' in sys.argv or len(sys.argv) == 1:
344             return
345
346         print('info: the build system for Rust is written in Rust, so this')
347         print('      script is now going to download a stage0 rust compiler')
348         print('      and then compile the build system itself')
349         print('')
350         print('info: in the meantime you can read more about rustbuild at')
351         print('      src/bootstrap/README.md before the download finishes')
352
353     def bootstrap_binary(self):
354         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
355
356     def build_bootstrap(self):
357         self.print_what_it_means_to_bootstrap()
358         build_dir = os.path.join(self.build_dir, "bootstrap")
359         if self.clean and os.path.exists(build_dir):
360             shutil.rmtree(build_dir)
361         env = os.environ.copy()
362         env["CARGO_TARGET_DIR"] = build_dir
363         env["RUSTC"] = self.rustc()
364         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
365                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
366                                  if "LD_LIBRARY_PATH" in env else ""
367         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
368                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
369                                    if "DYLD_LIBRARY_PATH" in env else ""
370         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
371                       os.pathsep + env["PATH"]
372         if not os.path.isfile(self.cargo()):
373             raise Exception("no cargo executable found at `%s`" % self.cargo())
374         args = [self.cargo(), "build", "--manifest-path",
375                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
376         if self.use_locked_deps:
377             args.append("--locked")
378         if self.use_vendored_sources:
379             args.append("--frozen")
380         self.run(args, env)
381
382     def run(self, args, env):
383         proc = subprocess.Popen(args, env=env)
384         ret = proc.wait()
385         if ret != 0:
386             sys.exit(ret)
387
388     def build_triple(self):
389         default_encoding = sys.getdefaultencoding()
390         config = self.get_toml('build')
391         if config:
392             return config
393         config = self.get_mk('CFG_BUILD')
394         if config:
395             return config
396         try:
397             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
398             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
399         except (subprocess.CalledProcessError, OSError):
400             if sys.platform == 'win32':
401                 return 'x86_64-pc-windows-msvc'
402             err = "uname not found"
403             if self.verbose:
404                 raise Exception(err)
405             sys.exit(err)
406
407         # The goal here is to come up with the same triple as LLVM would,
408         # at least for the subset of platforms we're willing to target.
409         if ostype == 'Linux':
410             os = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
411             if os == 'Android':
412                 ostype = 'linux-android'
413             else:
414                 ostype = 'unknown-linux-gnu'
415         elif ostype == 'FreeBSD':
416             ostype = 'unknown-freebsd'
417         elif ostype == 'DragonFly':
418             ostype = 'unknown-dragonfly'
419         elif ostype == 'Bitrig':
420             ostype = 'unknown-bitrig'
421         elif ostype == 'OpenBSD':
422             ostype = 'unknown-openbsd'
423         elif ostype == 'NetBSD':
424             ostype = 'unknown-netbsd'
425         elif ostype == 'SunOS':
426             ostype = 'sun-solaris'
427             # On Solaris, uname -m will return a machine classification instead
428             # of a cpu type, so uname -p is recommended instead.  However, the
429             # output from that option is too generic for our purposes (it will
430             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
431             # must be used instead.
432             try:
433                 cputype = subprocess.check_output(['isainfo',
434                   '-k']).strip().decode(default_encoding)
435             except (subprocess.CalledProcessError, OSError):
436                 err = "isainfo not found"
437                 if self.verbose:
438                     raise Exception(err)
439                 sys.exit(err)
440         elif ostype == 'Darwin':
441             ostype = 'apple-darwin'
442         elif ostype == 'Haiku':
443             ostype = 'unknown-haiku'
444         elif ostype.startswith('MINGW'):
445             # msys' `uname` does not print gcc configuration, but prints msys
446             # configuration. so we cannot believe `uname -m`:
447             # msys1 is always i686 and msys2 is always x86_64.
448             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
449             # MINGW64 on x86_64.
450             ostype = 'pc-windows-gnu'
451             cputype = 'i686'
452             if os.environ.get('MSYSTEM') == 'MINGW64':
453                 cputype = 'x86_64'
454         elif ostype.startswith('MSYS'):
455             ostype = 'pc-windows-gnu'
456         elif ostype.startswith('CYGWIN_NT'):
457             cputype = 'i686'
458             if ostype.endswith('WOW64'):
459                 cputype = 'x86_64'
460             ostype = 'pc-windows-gnu'
461         else:
462             err = "unknown OS type: " + ostype
463             if self.verbose:
464                 raise ValueError(err)
465             sys.exit(err)
466
467         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
468             cputype = 'i686'
469         elif cputype in {'xscale', 'arm'}:
470             cputype = 'arm'
471             if ostype == 'linux-android':
472                 ostype = 'linux-androideabi'
473         elif cputype == 'armv6l':
474             cputype = 'arm'
475             if ostype == 'linux-android':
476                 ostype = 'linux-androideabi'
477             else:
478                 ostype += 'eabihf'
479         elif cputype in {'armv7l', 'armv8l'}:
480             cputype = 'armv7'
481             if ostype == 'linux-android':
482                 ostype = 'linux-androideabi'
483             else:
484                 ostype += 'eabihf'
485         elif cputype in {'aarch64', 'arm64'}:
486             cputype = 'aarch64'
487         elif cputype == 'mips':
488             if sys.byteorder == 'big':
489                 cputype = 'mips'
490             elif sys.byteorder == 'little':
491                 cputype = 'mipsel'
492             else:
493                 raise ValueError('unknown byteorder: ' + sys.byteorder)
494         elif cputype == 'mips64':
495             if sys.byteorder == 'big':
496                 cputype = 'mips64'
497             elif sys.byteorder == 'little':
498                 cputype = 'mips64el'
499             else:
500                 raise ValueError('unknown byteorder: ' + sys.byteorder)
501             # only the n64 ABI is supported, indicate it
502             ostype += 'abi64'
503         elif cputype in {'powerpc', 'ppc'}:
504             cputype = 'powerpc'
505         elif cputype in {'powerpc64', 'ppc64'}:
506             cputype = 'powerpc64'
507         elif cputype in {'powerpc64le', 'ppc64le'}:
508             cputype = 'powerpc64le'
509         elif cputype == 'sparcv9':
510             pass
511         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
512             cputype = 'x86_64'
513         elif cputype == 's390x':
514             cputype = 's390x'
515         elif cputype == 'BePC':
516             cputype = 'i686'
517         else:
518             err = "unknown cpu type: " + cputype
519             if self.verbose:
520                 raise ValueError(err)
521             sys.exit(err)
522
523         return "{}-{}".format(cputype, ostype)
524
525 def bootstrap():
526     parser = argparse.ArgumentParser(description='Build rust')
527     parser.add_argument('--config')
528     parser.add_argument('--clean', action='store_true')
529     parser.add_argument('-v', '--verbose', action='store_true')
530
531     args = [a for a in sys.argv if a != '-h' and a != '--help']
532     args, _ = parser.parse_known_args(args)
533
534     # Configure initial bootstrap
535     rb = RustBuild()
536     rb.config_toml = ''
537     rb.config_mk = ''
538     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
539     rb.build_dir = os.path.join(os.getcwd(), "build")
540     rb.verbose = args.verbose
541     rb.clean = args.clean
542
543     try:
544         with open(args.config or 'config.toml') as config:
545             rb.config_toml = config.read()
546     except:
547         pass
548     try:
549         rb.config_mk = open('config.mk').read()
550     except:
551         pass
552
553     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
554                               'CFG_ENABLE_VENDOR' in rb.config_mk
555
556     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
557                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
558
559     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
560         if os.environ.get('USER') != os.environ['SUDO_USER']:
561             rb.use_vendored_sources = True
562             print('info: looks like you are running this command under `sudo`')
563             print('      and so in order to preserve your $HOME this will now')
564             print('      use vendored sources by default. Note that if this')
565             print('      does not work you should run a normal build first')
566             print('      before running a command like `sudo make install`')
567
568     if rb.use_vendored_sources:
569         if not os.path.exists('.cargo'):
570             os.makedirs('.cargo')
571         with open('.cargo/config','w') as f:
572             f.write("""
573                 [source.crates-io]
574                 replace-with = 'vendored-sources'
575                 registry = 'https://example.com'
576
577                 [source.vendored-sources]
578                 directory = '{}/src/vendor'
579             """.format(rb.rust_root))
580     else:
581         if os.path.exists('.cargo'):
582             shutil.rmtree('.cargo')
583
584     data = stage0_data(rb.rust_root)
585     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
586
587     # Fetch/build the bootstrap
588     rb.build = rb.build_triple()
589     rb.download_stage0()
590     sys.stdout.flush()
591     rb.build_bootstrap()
592     sys.stdout.flush()
593
594     # Run the bootstrap
595     args = [rb.bootstrap_binary()]
596     args.extend(sys.argv[1:])
597     env = os.environ.copy()
598     env["BUILD"] = rb.build
599     env["SRC"] = rb.rust_root
600     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
601     rb.run(args, env)
602
603 def main():
604     start_time = time()
605     help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
606     try:
607         bootstrap()
608         if not help_triggered:
609             print("Build completed successfully in %s" % format_build_time(time() - start_time))
610     except (SystemExit, KeyboardInterrupt) as e:
611         if hasattr(e, 'code') and isinstance(e.code, int):
612             exit_code = e.code
613         else:
614             exit_code = 1
615             print(e)
616         if not help_triggered:
617             print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
618         sys.exit(exit_code)
619
620 if __name__ == '__main__':
621     main()