]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
Rollup merge of #41456 - jessicah:haiku-support, r=alexcrichton
[rust.git] / src / bootstrap / bootstrap.py
1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
4 #
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
10
11 from __future__ import print_function
12 import argparse
13 import contextlib
14 import datetime
15 import hashlib
16 import os
17 import shutil
18 import subprocess
19 import sys
20 import tarfile
21 import tempfile
22
23 from time import time
24
25
26 def get(url, path, verbose=False):
27     sha_url = url + ".sha256"
28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
29         temp_path = temp_file.name
30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
31         sha_path = sha_file.name
32
33     try:
34         download(sha_path, sha_url, False, verbose)
35         if os.path.exists(path):
36             if verify(path, sha_path, False):
37                 if verbose:
38                     print("using already-download file " + path)
39                 return
40             else:
41                 if verbose:
42                     print("ignoring already-download file " + path + " due to failed verification")
43                 os.unlink(path)
44         download(temp_path, url, True, verbose)
45         if not verify(temp_path, sha_path, verbose):
46             raise RuntimeError("failed verification")
47         if verbose:
48             print("moving {} to {}".format(temp_path, path))
49         shutil.move(temp_path, path)
50     finally:
51         delete_if_present(sha_path, verbose)
52         delete_if_present(temp_path, verbose)
53
54
55 def delete_if_present(path, verbose):
56     if os.path.isfile(path):
57         if verbose:
58             print("removing " + path)
59         os.unlink(path)
60
61
62 def download(path, url, probably_big, verbose):
63     for x in range(0, 4):
64         try:
65             _download(path, url, probably_big, verbose, True)
66             return
67         except RuntimeError:
68             print("\nspurious failure, trying again")
69     _download(path, url, probably_big, verbose, False)
70
71
72 def _download(path, url, probably_big, verbose, exception):
73     if probably_big or verbose:
74         print("downloading {}".format(url))
75     # see http://serverfault.com/questions/301128/how-to-download
76     if sys.platform == 'win32':
77         run(["PowerShell.exe", "/nologo", "-Command",
78              "(New-Object System.Net.WebClient)"
79              ".DownloadFile('{}', '{}')".format(url, path)],
80             verbose=verbose,
81             exception=exception)
82     else:
83         if probably_big or verbose:
84             option = "-#"
85         else:
86             option = "-s"
87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
88             verbose=verbose,
89             exception=exception)
90
91
92 def verify(path, sha_path, verbose):
93     if verbose:
94         print("verifying " + path)
95     with open(path, "rb") as f:
96         found = hashlib.sha256(f.read()).hexdigest()
97     with open(sha_path, "r") as f:
98         expected = f.readline().split()[0]
99     verified = found == expected
100     if not verified:
101         print("invalid checksum:\n"
102                "    found:    {}\n"
103                "    expected: {}".format(found, expected))
104     return verified
105
106
107 def unpack(tarball, dst, verbose=False, match=None):
108     print("extracting " + tarball)
109     fname = os.path.basename(tarball).replace(".tar.gz", "")
110     with contextlib.closing(tarfile.open(tarball)) as tar:
111         for p in tar.getnames():
112             if "/" not in p:
113                 continue
114             name = p.replace(fname + "/", "", 1)
115             if match is not None and not name.startswith(match):
116                 continue
117             name = name[len(match) + 1:]
118
119             fp = os.path.join(dst, name)
120             if verbose:
121                 print("  extracting " + p)
122             tar.extract(p, dst)
123             tp = os.path.join(dst, p)
124             if os.path.isdir(tp) and os.path.exists(fp):
125                 continue
126             shutil.move(tp, fp)
127     shutil.rmtree(os.path.join(dst, fname))
128
129 def run(args, verbose=False, exception=False):
130     if verbose:
131         print("running: " + ' '.join(args))
132     sys.stdout.flush()
133     # Use Popen here instead of call() as it apparently allows powershell on
134     # Windows to not lock up waiting for input presumably.
135     ret = subprocess.Popen(args)
136     code = ret.wait()
137     if code != 0:
138         err = "failed to run: " + ' '.join(args)
139         if verbose or exception:
140             raise RuntimeError(err)
141         sys.exit(err)
142
143 def stage0_data(rust_root):
144     nightlies = os.path.join(rust_root, "src/stage0.txt")
145     data = {}
146     with open(nightlies, 'r') as nightlies:
147         for line in nightlies:
148             line = line.rstrip()  # Strip newline character, '\n'
149             if line.startswith("#") or line == '':
150                 continue
151             a, b = line.split(": ", 1)
152             data[a] = b
153     return data
154
155 def format_build_time(duration):
156     return str(datetime.timedelta(seconds=int(duration)))
157
158
159 class RustBuild(object):
160     def download_stage0(self):
161         cache_dst = os.path.join(self.build_dir, "cache")
162         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
163         if not os.path.exists(rustc_cache):
164             os.makedirs(rustc_cache)
165
166         channel = self.stage0_rustc_channel()
167
168         if self.rustc().startswith(self.bin_root()) and \
169                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
170             self.print_what_it_means_to_bootstrap()
171             if os.path.exists(self.bin_root()):
172                 shutil.rmtree(self.bin_root())
173             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
174             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
175             tarball = os.path.join(rustc_cache, filename)
176             if not os.path.exists(tarball):
177                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
178             unpack(tarball, self.bin_root(),
179                    match="rust-std-" + self.build,
180                    verbose=self.verbose)
181
182             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
183             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
184             tarball = os.path.join(rustc_cache, filename)
185             if not os.path.exists(tarball):
186                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
187             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
188             self.fix_executable(self.bin_root() + "/bin/rustc")
189             self.fix_executable(self.bin_root() + "/bin/rustdoc")
190             with open(self.rustc_stamp(), 'w') as f:
191                 f.write(self.stage0_rustc_date())
192
193             if "pc-windows-gnu" in self.build:
194                 filename = "rust-mingw-{}-{}.tar.gz".format(channel, self.build)
195                 url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
196                 tarball = os.path.join(rustc_cache, filename)
197                 if not os.path.exists(tarball):
198                     get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
199                 unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
200
201         if self.cargo().startswith(self.bin_root()) and \
202                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
203             self.print_what_it_means_to_bootstrap()
204             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
205             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
206             tarball = os.path.join(rustc_cache, filename)
207             if not os.path.exists(tarball):
208                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
209             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
210             self.fix_executable(self.bin_root() + "/bin/cargo")
211             with open(self.cargo_stamp(), 'w') as f:
212                 f.write(self.stage0_rustc_date())
213
214     def fix_executable(self, fname):
215         # If we're on NixOS we need to change the path to the dynamic loader
216
217         default_encoding = sys.getdefaultencoding()
218         try:
219             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
220         except (subprocess.CalledProcessError, WindowsError):
221             return
222
223         if ostype != "Linux":
224             return
225
226         if not os.path.exists("/etc/NIXOS"):
227             return
228         if os.path.exists("/lib"):
229             return
230
231         # At this point we're pretty sure the user is running NixOS
232         print("info: you seem to be running NixOS. Attempting to patch " + fname)
233
234         try:
235             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
236             interpreter = interpreter.strip().decode(default_encoding)
237         except subprocess.CalledProcessError as e:
238             print("warning: failed to call patchelf: %s" % e)
239             return
240
241         loader = interpreter.split("/")[-1]
242
243         try:
244             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
245             ldd_output = ldd_output.strip().decode(default_encoding)
246         except subprocess.CalledProcessError as e:
247             print("warning: unable to call ldd: %s" % e)
248             return
249
250         for line in ldd_output.splitlines():
251             libname = line.split()[0]
252             if libname.endswith(loader):
253                 loader_path = libname[:len(libname) - len(loader)]
254                 break
255         else:
256             print("warning: unable to find the path to the dynamic linker")
257             return
258
259         correct_interpreter = loader_path + loader
260
261         try:
262             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
263         except subprocess.CalledProcessError as e:
264             print("warning: failed to call patchelf: %s" % e)
265             return
266
267     def stage0_rustc_date(self):
268         return self._rustc_date
269
270     def stage0_rustc_channel(self):
271         return self._rustc_channel
272
273     def rustc_stamp(self):
274         return os.path.join(self.bin_root(), '.rustc-stamp')
275
276     def cargo_stamp(self):
277         return os.path.join(self.bin_root(), '.cargo-stamp')
278
279     def rustc_out_of_date(self):
280         if not os.path.exists(self.rustc_stamp()) or self.clean:
281             return True
282         with open(self.rustc_stamp(), 'r') as f:
283             return self.stage0_rustc_date() != f.read()
284
285     def cargo_out_of_date(self):
286         if not os.path.exists(self.cargo_stamp()) or self.clean:
287             return True
288         with open(self.cargo_stamp(), 'r') as f:
289             return self.stage0_rustc_date() != f.read()
290
291     def bin_root(self):
292         return os.path.join(self.build_dir, self.build, "stage0")
293
294     def get_toml(self, key):
295         for line in self.config_toml.splitlines():
296             if line.startswith(key + ' ='):
297                 return self.get_string(line)
298         return None
299
300     def get_mk(self, key):
301         for line in iter(self.config_mk.splitlines()):
302             if line.startswith(key + ' '):
303                 var = line[line.find(':=') + 2:].strip()
304                 if var != '':
305                     return var
306         return None
307
308     def cargo(self):
309         config = self.get_toml('cargo')
310         if config:
311             return config
312         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
313         if config:
314             return config + '/bin/cargo' + self.exe_suffix()
315         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
316
317     def rustc(self):
318         config = self.get_toml('rustc')
319         if config:
320             return config
321         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
322         if config:
323             return config + '/bin/rustc' + self.exe_suffix()
324         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
325
326     def get_string(self, line):
327         start = line.find('"')
328         end = start + 1 + line[start + 1:].find('"')
329         return line[start + 1:end]
330
331     def exe_suffix(self):
332         if sys.platform == 'win32':
333             return '.exe'
334         else:
335             return ''
336
337     def print_what_it_means_to_bootstrap(self):
338         if hasattr(self, 'printed'):
339             return
340         self.printed = True
341         if os.path.exists(self.bootstrap_binary()):
342             return
343         if not '--help' in sys.argv or len(sys.argv) == 1:
344             return
345
346         print('info: the build system for Rust is written in Rust, so this')
347         print('      script is now going to download a stage0 rust compiler')
348         print('      and then compile the build system itself')
349         print('')
350         print('info: in the meantime you can read more about rustbuild at')
351         print('      src/bootstrap/README.md before the download finishes')
352
353     def bootstrap_binary(self):
354         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
355
356     def build_bootstrap(self):
357         self.print_what_it_means_to_bootstrap()
358         build_dir = os.path.join(self.build_dir, "bootstrap")
359         if self.clean and os.path.exists(build_dir):
360             shutil.rmtree(build_dir)
361         env = os.environ.copy()
362         env["CARGO_TARGET_DIR"] = build_dir
363         env["RUSTC"] = self.rustc()
364         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
365                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
366                                  if "LD_LIBRARY_PATH" in env else ""
367         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
368                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
369                                    if "DYLD_LIBRARY_PATH" in env else ""
370         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
371                                    (os.pathsep + env["LIBRARY_PATH"]) \
372                                    if "LIBRARY_PATH" in env else ""
373         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
374                       os.pathsep + env["PATH"]
375         if not os.path.isfile(self.cargo()):
376             raise Exception("no cargo executable found at `%s`" % self.cargo())
377         args = [self.cargo(), "build", "--manifest-path",
378                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
379         if self.use_locked_deps:
380             args.append("--locked")
381         if self.use_vendored_sources:
382             args.append("--frozen")
383         self.run(args, env)
384
385     def run(self, args, env):
386         proc = subprocess.Popen(args, env=env)
387         ret = proc.wait()
388         if ret != 0:
389             sys.exit(ret)
390
391     def build_triple(self):
392         default_encoding = sys.getdefaultencoding()
393         config = self.get_toml('build')
394         if config:
395             return config
396         config = self.get_mk('CFG_BUILD')
397         if config:
398             return config
399         try:
400             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
401             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
402         except (subprocess.CalledProcessError, OSError):
403             if sys.platform == 'win32':
404                 return 'x86_64-pc-windows-msvc'
405             err = "uname not found"
406             if self.verbose:
407                 raise Exception(err)
408             sys.exit(err)
409
410         # The goal here is to come up with the same triple as LLVM would,
411         # at least for the subset of platforms we're willing to target.
412         if ostype == 'Linux':
413             os = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
414             if os == 'Android':
415                 ostype = 'linux-android'
416             else:
417                 ostype = 'unknown-linux-gnu'
418         elif ostype == 'FreeBSD':
419             ostype = 'unknown-freebsd'
420         elif ostype == 'DragonFly':
421             ostype = 'unknown-dragonfly'
422         elif ostype == 'Bitrig':
423             ostype = 'unknown-bitrig'
424         elif ostype == 'OpenBSD':
425             ostype = 'unknown-openbsd'
426         elif ostype == 'NetBSD':
427             ostype = 'unknown-netbsd'
428         elif ostype == 'SunOS':
429             ostype = 'sun-solaris'
430             # On Solaris, uname -m will return a machine classification instead
431             # of a cpu type, so uname -p is recommended instead.  However, the
432             # output from that option is too generic for our purposes (it will
433             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
434             # must be used instead.
435             try:
436                 cputype = subprocess.check_output(['isainfo',
437                   '-k']).strip().decode(default_encoding)
438             except (subprocess.CalledProcessError, OSError):
439                 err = "isainfo not found"
440                 if self.verbose:
441                     raise Exception(err)
442                 sys.exit(err)
443         elif ostype == 'Darwin':
444             ostype = 'apple-darwin'
445         elif ostype == 'Haiku':
446             ostype = 'unknown-haiku'
447         elif ostype.startswith('MINGW'):
448             # msys' `uname` does not print gcc configuration, but prints msys
449             # configuration. so we cannot believe `uname -m`:
450             # msys1 is always i686 and msys2 is always x86_64.
451             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
452             # MINGW64 on x86_64.
453             ostype = 'pc-windows-gnu'
454             cputype = 'i686'
455             if os.environ.get('MSYSTEM') == 'MINGW64':
456                 cputype = 'x86_64'
457         elif ostype.startswith('MSYS'):
458             ostype = 'pc-windows-gnu'
459         elif ostype.startswith('CYGWIN_NT'):
460             cputype = 'i686'
461             if ostype.endswith('WOW64'):
462                 cputype = 'x86_64'
463             ostype = 'pc-windows-gnu'
464         else:
465             err = "unknown OS type: " + ostype
466             if self.verbose:
467                 raise ValueError(err)
468             sys.exit(err)
469
470         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
471             cputype = 'i686'
472         elif cputype in {'xscale', 'arm'}:
473             cputype = 'arm'
474             if ostype == 'linux-android':
475                 ostype = 'linux-androideabi'
476         elif cputype == 'armv6l':
477             cputype = 'arm'
478             if ostype == 'linux-android':
479                 ostype = 'linux-androideabi'
480             else:
481                 ostype += 'eabihf'
482         elif cputype in {'armv7l', 'armv8l'}:
483             cputype = 'armv7'
484             if ostype == 'linux-android':
485                 ostype = 'linux-androideabi'
486             else:
487                 ostype += 'eabihf'
488         elif cputype in {'aarch64', 'arm64'}:
489             cputype = 'aarch64'
490         elif cputype == 'mips':
491             if sys.byteorder == 'big':
492                 cputype = 'mips'
493             elif sys.byteorder == 'little':
494                 cputype = 'mipsel'
495             else:
496                 raise ValueError('unknown byteorder: ' + sys.byteorder)
497         elif cputype == 'mips64':
498             if sys.byteorder == 'big':
499                 cputype = 'mips64'
500             elif sys.byteorder == 'little':
501                 cputype = 'mips64el'
502             else:
503                 raise ValueError('unknown byteorder: ' + sys.byteorder)
504             # only the n64 ABI is supported, indicate it
505             ostype += 'abi64'
506         elif cputype in {'powerpc', 'ppc'}:
507             cputype = 'powerpc'
508         elif cputype in {'powerpc64', 'ppc64'}:
509             cputype = 'powerpc64'
510         elif cputype in {'powerpc64le', 'ppc64le'}:
511             cputype = 'powerpc64le'
512         elif cputype == 'sparcv9':
513             pass
514         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
515             cputype = 'x86_64'
516         elif cputype == 's390x':
517             cputype = 's390x'
518         elif cputype == 'BePC':
519             cputype = 'i686'
520         else:
521             err = "unknown cpu type: " + cputype
522             if self.verbose:
523                 raise ValueError(err)
524             sys.exit(err)
525
526         return "{}-{}".format(cputype, ostype)
527
528 def bootstrap():
529     parser = argparse.ArgumentParser(description='Build rust')
530     parser.add_argument('--config')
531     parser.add_argument('--clean', action='store_true')
532     parser.add_argument('-v', '--verbose', action='store_true')
533
534     args = [a for a in sys.argv if a != '-h' and a != '--help']
535     args, _ = parser.parse_known_args(args)
536
537     # Configure initial bootstrap
538     rb = RustBuild()
539     rb.config_toml = ''
540     rb.config_mk = ''
541     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
542     rb.build_dir = os.path.join(os.getcwd(), "build")
543     rb.verbose = args.verbose
544     rb.clean = args.clean
545
546     try:
547         with open(args.config or 'config.toml') as config:
548             rb.config_toml = config.read()
549     except:
550         pass
551     try:
552         rb.config_mk = open('config.mk').read()
553     except:
554         pass
555
556     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
557                               'CFG_ENABLE_VENDOR' in rb.config_mk
558
559     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
560                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
561
562     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
563         if os.environ.get('USER') != os.environ['SUDO_USER']:
564             rb.use_vendored_sources = True
565             print('info: looks like you are running this command under `sudo`')
566             print('      and so in order to preserve your $HOME this will now')
567             print('      use vendored sources by default. Note that if this')
568             print('      does not work you should run a normal build first')
569             print('      before running a command like `sudo make install`')
570
571     if rb.use_vendored_sources:
572         if not os.path.exists('.cargo'):
573             os.makedirs('.cargo')
574         with open('.cargo/config','w') as f:
575             f.write("""
576                 [source.crates-io]
577                 replace-with = 'vendored-sources'
578                 registry = 'https://example.com'
579
580                 [source.vendored-sources]
581                 directory = '{}/src/vendor'
582             """.format(rb.rust_root))
583     else:
584         if os.path.exists('.cargo'):
585             shutil.rmtree('.cargo')
586
587     data = stage0_data(rb.rust_root)
588     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
589
590     # Fetch/build the bootstrap
591     rb.build = rb.build_triple()
592     rb.download_stage0()
593     sys.stdout.flush()
594     rb.build_bootstrap()
595     sys.stdout.flush()
596
597     # Run the bootstrap
598     args = [rb.bootstrap_binary()]
599     args.extend(sys.argv[1:])
600     env = os.environ.copy()
601     env["BUILD"] = rb.build
602     env["SRC"] = rb.rust_root
603     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
604     rb.run(args, env)
605
606 def main():
607     start_time = time()
608     help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
609     try:
610         bootstrap()
611         if not help_triggered:
612             print("Build completed successfully in %s" % format_build_time(time() - start_time))
613     except (SystemExit, KeyboardInterrupt) as e:
614         if hasattr(e, 'code') and isinstance(e.code, int):
615             exit_code = e.code
616         else:
617             exit_code = 1
618             print(e)
619         if not help_triggered:
620             print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
621         sys.exit(exit_code)
622
623 if __name__ == '__main__':
624     main()