]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/bootstrap.py
Haiku: add missing cases of using LIBRARY_PATH
[rust.git] / src / bootstrap / bootstrap.py
1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
4 #
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
10
11 from __future__ import print_function
12 import argparse
13 import contextlib
14 import datetime
15 import hashlib
16 import os
17 import shutil
18 import subprocess
19 import sys
20 import tarfile
21 import tempfile
22
23 from time import time
24
25
26 def get(url, path, verbose=False):
27     sha_url = url + ".sha256"
28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
29         temp_path = temp_file.name
30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
31         sha_path = sha_file.name
32
33     try:
34         download(sha_path, sha_url, False, verbose)
35         if os.path.exists(path):
36             if verify(path, sha_path, False):
37                 if verbose:
38                     print("using already-download file " + path)
39                 return
40             else:
41                 if verbose:
42                     print("ignoring already-download file " + path + " due to failed verification")
43                 os.unlink(path)
44         download(temp_path, url, True, verbose)
45         if not verify(temp_path, sha_path, verbose):
46             raise RuntimeError("failed verification")
47         if verbose:
48             print("moving {} to {}".format(temp_path, path))
49         shutil.move(temp_path, path)
50     finally:
51         delete_if_present(sha_path, verbose)
52         delete_if_present(temp_path, verbose)
53
54
55 def delete_if_present(path, verbose):
56     if os.path.isfile(path):
57         if verbose:
58             print("removing " + path)
59         os.unlink(path)
60
61
62 def download(path, url, probably_big, verbose):
63     for x in range(0, 4):
64         try:
65             _download(path, url, probably_big, verbose, True)
66             return
67         except RuntimeError:
68             print("\nspurious failure, trying again")
69     _download(path, url, probably_big, verbose, False)
70
71
72 def _download(path, url, probably_big, verbose, exception):
73     if probably_big or verbose:
74         print("downloading {}".format(url))
75     # see http://serverfault.com/questions/301128/how-to-download
76     if sys.platform == 'win32':
77         run(["PowerShell.exe", "/nologo", "-Command",
78              "(New-Object System.Net.WebClient)"
79              ".DownloadFile('{}', '{}')".format(url, path)],
80             verbose=verbose,
81             exception=exception)
82     else:
83         if probably_big or verbose:
84             option = "-#"
85         else:
86             option = "-s"
87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
88             verbose=verbose,
89             exception=exception)
90
91
92 def verify(path, sha_path, verbose):
93     if verbose:
94         print("verifying " + path)
95     with open(path, "rb") as f:
96         found = hashlib.sha256(f.read()).hexdigest()
97     with open(sha_path, "r") as f:
98         expected = f.readline().split()[0]
99     verified = found == expected
100     if not verified:
101         print("invalid checksum:\n"
102                "    found:    {}\n"
103                "    expected: {}".format(found, expected))
104     return verified
105
106
107 def unpack(tarball, dst, verbose=False, match=None):
108     print("extracting " + tarball)
109     fname = os.path.basename(tarball).replace(".tar.gz", "")
110     with contextlib.closing(tarfile.open(tarball)) as tar:
111         for p in tar.getnames():
112             if "/" not in p:
113                 continue
114             name = p.replace(fname + "/", "", 1)
115             if match is not None and not name.startswith(match):
116                 continue
117             name = name[len(match) + 1:]
118
119             fp = os.path.join(dst, name)
120             if verbose:
121                 print("  extracting " + p)
122             tar.extract(p, dst)
123             tp = os.path.join(dst, p)
124             if os.path.isdir(tp) and os.path.exists(fp):
125                 continue
126             shutil.move(tp, fp)
127     shutil.rmtree(os.path.join(dst, fname))
128
129 def run(args, verbose=False, exception=False):
130     if verbose:
131         print("running: " + ' '.join(args))
132     sys.stdout.flush()
133     # Use Popen here instead of call() as it apparently allows powershell on
134     # Windows to not lock up waiting for input presumably.
135     ret = subprocess.Popen(args)
136     code = ret.wait()
137     if code != 0:
138         err = "failed to run: " + ' '.join(args)
139         if verbose or exception:
140             raise RuntimeError(err)
141         sys.exit(err)
142
143 def stage0_data(rust_root):
144     nightlies = os.path.join(rust_root, "src/stage0.txt")
145     data = {}
146     with open(nightlies, 'r') as nightlies:
147         for line in nightlies:
148             line = line.rstrip()  # Strip newline character, '\n'
149             if line.startswith("#") or line == '':
150                 continue
151             a, b = line.split(": ", 1)
152             data[a] = b
153     return data
154
155 def format_build_time(duration):
156     return str(datetime.timedelta(seconds=int(duration)))
157
158
159 class RustBuild(object):
160     def download_stage0(self):
161         cache_dst = os.path.join(self.build_dir, "cache")
162         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
163         if not os.path.exists(rustc_cache):
164             os.makedirs(rustc_cache)
165
166         channel = self.stage0_rustc_channel()
167
168         if self.rustc().startswith(self.bin_root()) and \
169                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
170             self.print_what_it_means_to_bootstrap()
171             if os.path.exists(self.bin_root()):
172                 shutil.rmtree(self.bin_root())
173             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
174             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
175             tarball = os.path.join(rustc_cache, filename)
176             if not os.path.exists(tarball):
177                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
178             unpack(tarball, self.bin_root(),
179                    match="rust-std-" + self.build,
180                    verbose=self.verbose)
181
182             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
183             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
184             tarball = os.path.join(rustc_cache, filename)
185             if not os.path.exists(tarball):
186                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
187             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
188             self.fix_executable(self.bin_root() + "/bin/rustc")
189             self.fix_executable(self.bin_root() + "/bin/rustdoc")
190             with open(self.rustc_stamp(), 'w') as f:
191                 f.write(self.stage0_rustc_date())
192
193             if "pc-windows-gnu" in self.build:
194                 filename = "rust-mingw-{}-{}.tar.gz".format(channel, self.build)
195                 url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
196                 tarball = os.path.join(rustc_cache, filename)
197                 if not os.path.exists(tarball):
198                     get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
199                 unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
200
201         if self.cargo().startswith(self.bin_root()) and \
202                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
203             self.print_what_it_means_to_bootstrap()
204             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
205             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
206             tarball = os.path.join(rustc_cache, filename)
207             if not os.path.exists(tarball):
208                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
209             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
210             self.fix_executable(self.bin_root() + "/bin/cargo")
211             with open(self.cargo_stamp(), 'w') as f:
212                 f.write(self.stage0_rustc_date())
213
214     def fix_executable(self, fname):
215         # If we're on NixOS we need to change the path to the dynamic loader
216
217         default_encoding = sys.getdefaultencoding()
218         try:
219             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
220         except (subprocess.CalledProcessError, WindowsError):
221             return
222
223         if ostype != "Linux":
224             return
225
226         if not os.path.exists("/etc/NIXOS"):
227             return
228         if os.path.exists("/lib"):
229             return
230
231         # At this point we're pretty sure the user is running NixOS
232         print("info: you seem to be running NixOS. Attempting to patch " + fname)
233
234         try:
235             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
236             interpreter = interpreter.strip().decode(default_encoding)
237         except subprocess.CalledProcessError as e:
238             print("warning: failed to call patchelf: %s" % e)
239             return
240
241         loader = interpreter.split("/")[-1]
242
243         try:
244             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
245             ldd_output = ldd_output.strip().decode(default_encoding)
246         except subprocess.CalledProcessError as e:
247             print("warning: unable to call ldd: %s" % e)
248             return
249
250         for line in ldd_output.splitlines():
251             libname = line.split()[0]
252             if libname.endswith(loader):
253                 loader_path = libname[:len(libname) - len(loader)]
254                 break
255         else:
256             print("warning: unable to find the path to the dynamic linker")
257             return
258
259         correct_interpreter = loader_path + loader
260
261         try:
262             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
263         except subprocess.CalledProcessError as e:
264             print("warning: failed to call patchelf: %s" % e)
265             return
266
267     def stage0_rustc_date(self):
268         return self._rustc_date
269
270     def stage0_rustc_channel(self):
271         return self._rustc_channel
272
273     def rustc_stamp(self):
274         return os.path.join(self.bin_root(), '.rustc-stamp')
275
276     def cargo_stamp(self):
277         return os.path.join(self.bin_root(), '.cargo-stamp')
278
279     def rustc_out_of_date(self):
280         if not os.path.exists(self.rustc_stamp()) or self.clean:
281             return True
282         with open(self.rustc_stamp(), 'r') as f:
283             return self.stage0_rustc_date() != f.read()
284
285     def cargo_out_of_date(self):
286         if not os.path.exists(self.cargo_stamp()) or self.clean:
287             return True
288         with open(self.cargo_stamp(), 'r') as f:
289             return self.stage0_rustc_date() != f.read()
290
291     def bin_root(self):
292         return os.path.join(self.build_dir, self.build, "stage0")
293
294     def get_toml(self, key):
295         for line in self.config_toml.splitlines():
296             if line.startswith(key + ' ='):
297                 return self.get_string(line)
298         return None
299
300     def get_mk(self, key):
301         for line in iter(self.config_mk.splitlines()):
302             if line.startswith(key + ' '):
303                 var = line[line.find(':=') + 2:].strip()
304                 if var != '':
305                     return var
306         return None
307
308     def cargo(self):
309         config = self.get_toml('cargo')
310         if config:
311             return config
312         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
313         if config:
314             return config + '/bin/cargo' + self.exe_suffix()
315         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
316
317     def rustc(self):
318         config = self.get_toml('rustc')
319         if config:
320             return config
321         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
322         if config:
323             return config + '/bin/rustc' + self.exe_suffix()
324         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
325
326     def get_string(self, line):
327         start = line.find('"')
328         end = start + 1 + line[start + 1:].find('"')
329         return line[start + 1:end]
330
331     def exe_suffix(self):
332         if sys.platform == 'win32':
333             return '.exe'
334         else:
335             return ''
336
337     def print_what_it_means_to_bootstrap(self):
338         if hasattr(self, 'printed'):
339             return
340         self.printed = True
341         if os.path.exists(self.bootstrap_binary()):
342             return
343         if not '--help' in sys.argv or len(sys.argv) == 1:
344             return
345
346         print('info: the build system for Rust is written in Rust, so this')
347         print('      script is now going to download a stage0 rust compiler')
348         print('      and then compile the build system itself')
349         print('')
350         print('info: in the meantime you can read more about rustbuild at')
351         print('      src/bootstrap/README.md before the download finishes')
352
353     def bootstrap_binary(self):
354         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
355
356     def build_bootstrap(self):
357         self.print_what_it_means_to_bootstrap()
358         build_dir = os.path.join(self.build_dir, "bootstrap")
359         if self.clean and os.path.exists(build_dir):
360             shutil.rmtree(build_dir)
361         env = os.environ.copy()
362         env["CARGO_TARGET_DIR"] = build_dir
363         env["RUSTC"] = self.rustc()
364         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
365                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
366                                  if "LD_LIBRARY_PATH" in env else ""
367         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
368                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
369                                    if "DYLD_LIBRARY_PATH" in env else ""
370         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
371                                    (os.pathsep + env["LIBRARY_PATH"]) \
372                                    if "LIBRARY_PATH" in env else ""
373         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
374                       os.pathsep + env["PATH"]
375         if not os.path.isfile(self.cargo()):
376             raise Exception("no cargo executable found at `%s`" % self.cargo())
377         args = [self.cargo(), "build", "--manifest-path",
378                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
379         if self.use_locked_deps:
380             args.append("--locked")
381         if self.use_vendored_sources:
382             args.append("--frozen")
383         self.run(args, env)
384
385     def run(self, args, env):
386         proc = subprocess.Popen(args, env=env)
387         ret = proc.wait()
388         if ret != 0:
389             sys.exit(ret)
390
391     def build_triple(self):
392         default_encoding = sys.getdefaultencoding()
393         config = self.get_toml('build')
394         if config:
395             return config
396         config = self.get_mk('CFG_BUILD')
397         if config:
398             return config
399         try:
400             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
401             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
402         except (subprocess.CalledProcessError, OSError):
403             if sys.platform == 'win32':
404                 return 'x86_64-pc-windows-msvc'
405             err = "uname not found"
406             if self.verbose:
407                 raise Exception(err)
408             sys.exit(err)
409
410         # The goal here is to come up with the same triple as LLVM would,
411         # at least for the subset of platforms we're willing to target.
412         if ostype == 'Linux':
413             ostype = 'unknown-linux-gnu'
414         elif ostype == 'FreeBSD':
415             ostype = 'unknown-freebsd'
416         elif ostype == 'DragonFly':
417             ostype = 'unknown-dragonfly'
418         elif ostype == 'Bitrig':
419             ostype = 'unknown-bitrig'
420         elif ostype == 'OpenBSD':
421             ostype = 'unknown-openbsd'
422         elif ostype == 'NetBSD':
423             ostype = 'unknown-netbsd'
424         elif ostype == 'SunOS':
425             ostype = 'sun-solaris'
426             # On Solaris, uname -m will return a machine classification instead
427             # of a cpu type, so uname -p is recommended instead.  However, the
428             # output from that option is too generic for our purposes (it will
429             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
430             # must be used instead.
431             try:
432                 cputype = subprocess.check_output(['isainfo',
433                   '-k']).strip().decode(default_encoding)
434             except (subprocess.CalledProcessError, OSError):
435                 err = "isainfo not found"
436                 if self.verbose:
437                     raise Exception(err)
438                 sys.exit(err)
439         elif ostype == 'Darwin':
440             ostype = 'apple-darwin'
441         elif ostype == 'Haiku':
442             ostype = 'unknown-haiku'
443         elif ostype.startswith('MINGW'):
444             # msys' `uname` does not print gcc configuration, but prints msys
445             # configuration. so we cannot believe `uname -m`:
446             # msys1 is always i686 and msys2 is always x86_64.
447             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
448             # MINGW64 on x86_64.
449             ostype = 'pc-windows-gnu'
450             cputype = 'i686'
451             if os.environ.get('MSYSTEM') == 'MINGW64':
452                 cputype = 'x86_64'
453         elif ostype.startswith('MSYS'):
454             ostype = 'pc-windows-gnu'
455         elif ostype.startswith('CYGWIN_NT'):
456             cputype = 'i686'
457             if ostype.endswith('WOW64'):
458                 cputype = 'x86_64'
459             ostype = 'pc-windows-gnu'
460         else:
461             err = "unknown OS type: " + ostype
462             if self.verbose:
463                 raise ValueError(err)
464             sys.exit(err)
465
466         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
467             cputype = 'i686'
468         elif cputype in {'xscale', 'arm'}:
469             cputype = 'arm'
470         elif cputype == 'armv6l':
471             cputype = 'arm'
472             ostype += 'eabihf'
473         elif cputype in {'armv7l', 'armv8l'}:
474             cputype = 'armv7'
475             ostype += 'eabihf'
476         elif cputype == 'aarch64':
477             cputype = 'aarch64'
478         elif cputype == 'arm64':
479             cputype = 'aarch64'
480         elif cputype == 'mips':
481             if sys.byteorder == 'big':
482                 cputype = 'mips'
483             elif sys.byteorder == 'little':
484                 cputype = 'mipsel'
485             else:
486                 raise ValueError('unknown byteorder: ' + sys.byteorder)
487         elif cputype == 'mips64':
488             if sys.byteorder == 'big':
489                 cputype = 'mips64'
490             elif sys.byteorder == 'little':
491                 cputype = 'mips64el'
492             else:
493                 raise ValueError('unknown byteorder: ' + sys.byteorder)
494             # only the n64 ABI is supported, indicate it
495             ostype += 'abi64'
496         elif cputype in {'powerpc', 'ppc'}:
497             cputype = 'powerpc'
498         elif cputype in {'powerpc64', 'ppc64'}:
499             cputype = 'powerpc64'
500         elif cputype in {'powerpc64le', 'ppc64le'}:
501             cputype = 'powerpc64le'
502         elif cputype == 'sparcv9':
503             pass
504         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
505             cputype = 'x86_64'
506         elif cputype == 's390x':
507             cputype = 's390x'
508         elif cputype == 'BePC':
509             cputype = 'i686'
510         else:
511             err = "unknown cpu type: " + cputype
512             if self.verbose:
513                 raise ValueError(err)
514             sys.exit(err)
515
516         return "{}-{}".format(cputype, ostype)
517
518 def bootstrap():
519     parser = argparse.ArgumentParser(description='Build rust')
520     parser.add_argument('--config')
521     parser.add_argument('--clean', action='store_true')
522     parser.add_argument('-v', '--verbose', action='store_true')
523
524     args = [a for a in sys.argv if a != '-h' and a != '--help']
525     args, _ = parser.parse_known_args(args)
526
527     # Configure initial bootstrap
528     rb = RustBuild()
529     rb.config_toml = ''
530     rb.config_mk = ''
531     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
532     rb.build_dir = os.path.join(os.getcwd(), "build")
533     rb.verbose = args.verbose
534     rb.clean = args.clean
535
536     try:
537         with open(args.config or 'config.toml') as config:
538             rb.config_toml = config.read()
539     except:
540         pass
541     try:
542         rb.config_mk = open('config.mk').read()
543     except:
544         pass
545
546     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
547                               'CFG_ENABLE_VENDOR' in rb.config_mk
548
549     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
550                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
551
552     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
553         if os.environ.get('USER') != os.environ['SUDO_USER']:
554             rb.use_vendored_sources = True
555             print('info: looks like you are running this command under `sudo`')
556             print('      and so in order to preserve your $HOME this will now')
557             print('      use vendored sources by default. Note that if this')
558             print('      does not work you should run a normal build first')
559             print('      before running a command like `sudo make install`')
560
561     if rb.use_vendored_sources:
562         if not os.path.exists('.cargo'):
563             os.makedirs('.cargo')
564         with open('.cargo/config','w') as f:
565             f.write("""
566                 [source.crates-io]
567                 replace-with = 'vendored-sources'
568                 registry = 'https://example.com'
569
570                 [source.vendored-sources]
571                 directory = '{}/src/vendor'
572             """.format(rb.rust_root))
573     else:
574         if os.path.exists('.cargo'):
575             shutil.rmtree('.cargo')
576
577     data = stage0_data(rb.rust_root)
578     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
579
580     # Fetch/build the bootstrap
581     rb.build = rb.build_triple()
582     rb.download_stage0()
583     sys.stdout.flush()
584     rb.build_bootstrap()
585     sys.stdout.flush()
586
587     # Run the bootstrap
588     args = [rb.bootstrap_binary()]
589     args.extend(sys.argv[1:])
590     env = os.environ.copy()
591     env["BUILD"] = rb.build
592     env["SRC"] = rb.rust_root
593     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
594     rb.run(args, env)
595
596 def main():
597     start_time = time()
598     help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
599     try:
600         bootstrap()
601         if not help_triggered:
602             print("Build completed successfully in %s" % format_build_time(time() - start_time))
603     except (SystemExit, KeyboardInterrupt) as e:
604         if hasattr(e, 'code') and isinstance(e.code, int):
605             exit_code = e.code
606         else:
607             exit_code = 1
608             print(e)
609         if not help_triggered:
610             print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
611         sys.exit(exit_code)
612
613 if __name__ == '__main__':
614     main()