1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
11 from __future__ import print_function
27 def get(url, path, verbose=False):
28 sha_url = url + ".sha256"
29 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
30 temp_path = temp_file.name
31 with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
32 sha_path = sha_file.name
35 download(sha_path, sha_url, False, verbose)
36 if os.path.exists(path):
37 if verify(path, sha_path, False):
39 print("using already-download file " + path)
43 print("ignoring already-download file " +
44 path + " due to failed verification")
46 download(temp_path, url, True, verbose)
47 if not verify(temp_path, sha_path, verbose):
48 raise RuntimeError("failed verification")
50 print("moving {} to {}".format(temp_path, path))
51 shutil.move(temp_path, path)
53 delete_if_present(sha_path, verbose)
54 delete_if_present(temp_path, verbose)
57 def delete_if_present(path, verbose):
58 if os.path.isfile(path):
60 print("removing " + path)
64 def download(path, url, probably_big, verbose):
67 _download(path, url, probably_big, verbose, True)
70 print("\nspurious failure, trying again")
71 _download(path, url, probably_big, verbose, False)
74 def _download(path, url, probably_big, verbose, exception):
75 if probably_big or verbose:
76 print("downloading {}".format(url))
77 # see http://serverfault.com/questions/301128/how-to-download
78 if sys.platform == 'win32':
79 run(["PowerShell.exe", "/nologo", "-Command",
80 "(New-Object System.Net.WebClient)"
81 ".DownloadFile('{}', '{}')".format(url, path)],
85 if probably_big or verbose:
89 run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
94 def verify(path, sha_path, verbose):
96 print("verifying " + path)
97 with open(path, "rb") as f:
98 found = hashlib.sha256(f.read()).hexdigest()
99 with open(sha_path, "r") as f:
100 expected = f.readline().split()[0]
101 verified = found == expected
103 print("invalid checksum:\n"
105 " expected: {}".format(found, expected))
109 def unpack(tarball, dst, verbose=False, match=None):
110 print("extracting " + tarball)
111 fname = os.path.basename(tarball).replace(".tar.gz", "")
112 with contextlib.closing(tarfile.open(tarball)) as tar:
113 for p in tar.getnames():
116 name = p.replace(fname + "/", "", 1)
117 if match is not None and not name.startswith(match):
119 name = name[len(match) + 1:]
121 fp = os.path.join(dst, name)
123 print(" extracting " + p)
125 tp = os.path.join(dst, p)
126 if os.path.isdir(tp) and os.path.exists(fp):
129 shutil.rmtree(os.path.join(dst, fname))
131 def run(args, verbose=False, exception=False, **kwargs):
133 print("running: " + ' '.join(args))
135 # Use Popen here instead of call() as it apparently allows powershell on
136 # Windows to not lock up waiting for input presumably.
137 ret = subprocess.Popen(args, **kwargs)
140 err = "failed to run: " + ' '.join(args)
141 if verbose or exception:
142 raise RuntimeError(err)
146 def stage0_data(rust_root):
147 nightlies = os.path.join(rust_root, "src/stage0.txt")
149 with open(nightlies, 'r') as nightlies:
150 for line in nightlies:
151 line = line.rstrip() # Strip newline character, '\n'
152 if line.startswith("#") or line == '':
154 a, b = line.split(": ", 1)
159 def format_build_time(duration):
160 return str(datetime.timedelta(seconds=int(duration)))
163 class RustBuild(object):
165 def download_stage0(self):
166 cache_dst = os.path.join(self.build_dir, "cache")
167 rustc_cache = os.path.join(cache_dst, self.stage0_date())
168 if not os.path.exists(rustc_cache):
169 os.makedirs(rustc_cache)
171 rustc_channel = self.stage0_rustc_channel()
172 cargo_channel = self.stage0_cargo_channel()
174 if self.rustc().startswith(self.bin_root()) and \
175 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
176 self.print_what_it_means_to_bootstrap()
177 if os.path.exists(self.bin_root()):
178 shutil.rmtree(self.bin_root())
179 filename = "rust-std-{}-{}.tar.gz".format(
180 rustc_channel, self.build)
181 url = self._download_url + "/dist/" + self.stage0_date()
182 tarball = os.path.join(rustc_cache, filename)
183 if not os.path.exists(tarball):
184 get("{}/{}".format(url, filename),
185 tarball, verbose=self.verbose)
186 unpack(tarball, self.bin_root(),
187 match="rust-std-" + self.build,
188 verbose=self.verbose)
190 filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
191 url = self._download_url + "/dist/" + self.stage0_date()
192 tarball = os.path.join(rustc_cache, filename)
193 if not os.path.exists(tarball):
194 get("{}/{}".format(url, filename),
195 tarball, verbose=self.verbose)
196 unpack(tarball, self.bin_root(),
197 match="rustc", verbose=self.verbose)
198 self.fix_executable(self.bin_root() + "/bin/rustc")
199 self.fix_executable(self.bin_root() + "/bin/rustdoc")
200 with open(self.rustc_stamp(), 'w') as f:
201 f.write(self.stage0_date())
203 if "pc-windows-gnu" in self.build:
204 filename = "rust-mingw-{}-{}.tar.gz".format(
205 rustc_channel, self.build)
206 url = self._download_url + "/dist/" + self.stage0_date()
207 tarball = os.path.join(rustc_cache, filename)
208 if not os.path.exists(tarball):
209 get("{}/{}".format(url, filename),
210 tarball, verbose=self.verbose)
211 unpack(tarball, self.bin_root(),
212 match="rust-mingw", verbose=self.verbose)
214 if self.cargo().startswith(self.bin_root()) and \
215 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
216 self.print_what_it_means_to_bootstrap()
217 filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
218 url = self._download_url + "/dist/" + self.stage0_date()
219 tarball = os.path.join(rustc_cache, filename)
220 if not os.path.exists(tarball):
221 get("{}/{}".format(url, filename),
222 tarball, verbose=self.verbose)
223 unpack(tarball, self.bin_root(),
224 match="cargo", verbose=self.verbose)
225 self.fix_executable(self.bin_root() + "/bin/cargo")
226 with open(self.cargo_stamp(), 'w') as f:
227 f.write(self.stage0_date())
229 def fix_executable(self, fname):
230 # If we're on NixOS we need to change the path to the dynamic loader
232 default_encoding = sys.getdefaultencoding()
234 ostype = subprocess.check_output(
235 ['uname', '-s']).strip().decode(default_encoding)
236 except (subprocess.CalledProcessError, WindowsError):
239 if ostype != "Linux":
242 if not os.path.exists("/etc/NIXOS"):
244 if os.path.exists("/lib"):
247 # At this point we're pretty sure the user is running NixOS
248 print("info: you seem to be running NixOS. Attempting to patch " + fname)
251 interpreter = subprocess.check_output(
252 ["patchelf", "--print-interpreter", fname])
253 interpreter = interpreter.strip().decode(default_encoding)
254 except subprocess.CalledProcessError as e:
255 print("warning: failed to call patchelf: %s" % e)
258 loader = interpreter.split("/")[-1]
261 ldd_output = subprocess.check_output(
262 ['ldd', '/run/current-system/sw/bin/sh'])
263 ldd_output = ldd_output.strip().decode(default_encoding)
264 except subprocess.CalledProcessError as e:
265 print("warning: unable to call ldd: %s" % e)
268 for line in ldd_output.splitlines():
269 libname = line.split()[0]
270 if libname.endswith(loader):
271 loader_path = libname[:len(libname) - len(loader)]
274 print("warning: unable to find the path to the dynamic linker")
277 correct_interpreter = loader_path + loader
280 subprocess.check_output(
281 ["patchelf", "--set-interpreter", correct_interpreter, fname])
282 except subprocess.CalledProcessError as e:
283 print("warning: failed to call patchelf: %s" % e)
286 def stage0_date(self):
289 def stage0_rustc_channel(self):
290 return self._rustc_channel
292 def stage0_cargo_channel(self):
293 return self._cargo_channel
295 def rustc_stamp(self):
296 return os.path.join(self.bin_root(), '.rustc-stamp')
298 def cargo_stamp(self):
299 return os.path.join(self.bin_root(), '.cargo-stamp')
301 def rustc_out_of_date(self):
302 if not os.path.exists(self.rustc_stamp()) or self.clean:
304 with open(self.rustc_stamp(), 'r') as f:
305 return self.stage0_date() != f.read()
307 def cargo_out_of_date(self):
308 if not os.path.exists(self.cargo_stamp()) or self.clean:
310 with open(self.cargo_stamp(), 'r') as f:
311 return self.stage0_date() != f.read()
314 return os.path.join(self.build_dir, self.build, "stage0")
316 def get_toml(self, key):
317 for line in self.config_toml.splitlines():
318 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
319 if match is not None:
320 value = match.group(1)
321 return self.get_string(value) or value.strip()
324 def get_mk(self, key):
325 for line in iter(self.config_mk.splitlines()):
326 if line.startswith(key + ' '):
327 var = line[line.find(':=') + 2:].strip()
333 config = self.get_toml('cargo')
336 config = self.get_mk('CFG_LOCAL_RUST_ROOT')
338 return config + '/bin/cargo' + self.exe_suffix()
339 return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
342 config = self.get_toml('rustc')
345 config = self.get_mk('CFG_LOCAL_RUST_ROOT')
347 return config + '/bin/rustc' + self.exe_suffix()
348 return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
350 def get_string(self, line):
351 start = line.find('"')
354 end = start + 1 + line[start + 1:].find('"')
355 return line[start + 1:end]
357 def exe_suffix(self):
358 if sys.platform == 'win32':
363 def print_what_it_means_to_bootstrap(self):
364 if hasattr(self, 'printed'):
367 if os.path.exists(self.bootstrap_binary()):
369 if not '--help' in sys.argv or len(sys.argv) == 1:
372 print('info: the build system for Rust is written in Rust, so this')
373 print(' script is now going to download a stage0 rust compiler')
374 print(' and then compile the build system itself')
376 print('info: in the meantime you can read more about rustbuild at')
377 print(' src/bootstrap/README.md before the download finishes')
379 def bootstrap_binary(self):
380 return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
382 def build_bootstrap(self):
383 self.print_what_it_means_to_bootstrap()
384 build_dir = os.path.join(self.build_dir, "bootstrap")
385 if self.clean and os.path.exists(build_dir):
386 shutil.rmtree(build_dir)
387 env = os.environ.copy()
388 env["CARGO_TARGET_DIR"] = build_dir
389 env["RUSTC"] = self.rustc()
390 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
391 (os.pathsep + env["LD_LIBRARY_PATH"]) \
392 if "LD_LIBRARY_PATH" in env else ""
393 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
394 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
395 if "DYLD_LIBRARY_PATH" in env else ""
396 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
397 (os.pathsep + env["LIBRARY_PATH"]) \
398 if "LIBRARY_PATH" in env else ""
399 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
400 os.pathsep + env["PATH"]
401 if not os.path.isfile(self.cargo()):
402 raise Exception("no cargo executable found at `%s`" % self.cargo())
403 args = [self.cargo(), "build", "--manifest-path",
404 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
406 args.append("--verbose")
408 args.append("--verbose")
409 if self.use_locked_deps:
410 args.append("--locked")
411 if self.use_vendored_sources:
412 args.append("--frozen")
413 run(args, env=env, verbose=self.verbose)
415 def build_triple(self):
416 default_encoding = sys.getdefaultencoding()
417 config = self.get_toml('build')
420 config = self.get_mk('CFG_BUILD')
424 ostype = subprocess.check_output(
425 ['uname', '-s']).strip().decode(default_encoding)
426 cputype = subprocess.check_output(
427 ['uname', '-m']).strip().decode(default_encoding)
428 except (subprocess.CalledProcessError, OSError):
429 if sys.platform == 'win32':
430 return 'x86_64-pc-windows-msvc'
431 err = "uname not found"
436 # The goal here is to come up with the same triple as LLVM would,
437 # at least for the subset of platforms we're willing to target.
438 if ostype == 'Linux':
439 os_from_sp = subprocess.check_output(
440 ['uname', '-o']).strip().decode(default_encoding)
441 if os_from_sp == 'Android':
442 ostype = 'linux-android'
444 ostype = 'unknown-linux-gnu'
445 elif ostype == 'FreeBSD':
446 ostype = 'unknown-freebsd'
447 elif ostype == 'DragonFly':
448 ostype = 'unknown-dragonfly'
449 elif ostype == 'Bitrig':
450 ostype = 'unknown-bitrig'
451 elif ostype == 'OpenBSD':
452 ostype = 'unknown-openbsd'
453 elif ostype == 'NetBSD':
454 ostype = 'unknown-netbsd'
455 elif ostype == 'SunOS':
456 ostype = 'sun-solaris'
457 # On Solaris, uname -m will return a machine classification instead
458 # of a cpu type, so uname -p is recommended instead. However, the
459 # output from that option is too generic for our purposes (it will
460 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
461 # must be used instead.
463 cputype = subprocess.check_output(['isainfo',
464 '-k']).strip().decode(default_encoding)
465 except (subprocess.CalledProcessError, OSError):
466 err = "isainfo not found"
470 elif ostype == 'Darwin':
471 ostype = 'apple-darwin'
472 elif ostype == 'Haiku':
473 ostype = 'unknown-haiku'
474 elif ostype.startswith('MINGW'):
475 # msys' `uname` does not print gcc configuration, but prints msys
476 # configuration. so we cannot believe `uname -m`:
477 # msys1 is always i686 and msys2 is always x86_64.
478 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
480 ostype = 'pc-windows-gnu'
482 if os.environ.get('MSYSTEM') == 'MINGW64':
484 elif ostype.startswith('MSYS'):
485 ostype = 'pc-windows-gnu'
486 elif ostype.startswith('CYGWIN_NT'):
488 if ostype.endswith('WOW64'):
490 ostype = 'pc-windows-gnu'
492 err = "unknown OS type: " + ostype
494 raise ValueError(err)
497 if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
499 elif cputype in {'xscale', 'arm'}:
501 if ostype == 'linux-android':
502 ostype = 'linux-androideabi'
503 elif cputype == 'armv6l':
505 if ostype == 'linux-android':
506 ostype = 'linux-androideabi'
509 elif cputype in {'armv7l', 'armv8l'}:
511 if ostype == 'linux-android':
512 ostype = 'linux-androideabi'
515 elif cputype in {'aarch64', 'arm64'}:
517 elif cputype == 'mips':
518 if sys.byteorder == 'big':
520 elif sys.byteorder == 'little':
523 raise ValueError('unknown byteorder: ' + sys.byteorder)
524 elif cputype == 'mips64':
525 if sys.byteorder == 'big':
527 elif sys.byteorder == 'little':
530 raise ValueError('unknown byteorder: ' + sys.byteorder)
531 # only the n64 ABI is supported, indicate it
533 elif cputype in {'powerpc', 'ppc'}:
535 elif cputype in {'powerpc64', 'ppc64'}:
536 cputype = 'powerpc64'
537 elif cputype in {'powerpc64le', 'ppc64le'}:
538 cputype = 'powerpc64le'
539 elif cputype == 'sparcv9':
541 elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
543 elif cputype == 's390x':
545 elif cputype == 'BePC':
548 err = "unknown cpu type: " + cputype
550 raise ValueError(err)
553 return "{}-{}".format(cputype, ostype)
555 def update_submodules(self):
556 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
557 self.get_toml('submodules') == "false" or \
558 self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
560 print('Updating submodules')
561 default_encoding = sys.getdefaultencoding()
562 run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
563 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
564 ["git", "config", "--file", os.path.join(self.rust_root, ".gitmodules"),
565 "--get-regexp", "path"]
566 ).decode(default_encoding).splitlines()]
567 submodules = [module for module in submodules
568 if not ((module.endswith("llvm") and
569 (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT'))) or
570 (module.endswith("jemalloc") and
571 (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT'))))
573 run(["git", "submodule", "update",
574 "--init"] + submodules, cwd=self.rust_root)
575 run(["git", "submodule", "-q", "foreach", "git",
576 "reset", "-q", "--hard"], cwd=self.rust_root)
577 run(["git", "submodule", "-q", "foreach", "git",
578 "clean", "-qdfx"], cwd=self.rust_root)
582 parser = argparse.ArgumentParser(description='Build rust')
583 parser.add_argument('--config')
584 parser.add_argument('--clean', action='store_true')
585 parser.add_argument('-v', '--verbose', action='store_true')
587 args = [a for a in sys.argv if a != '-h' and a != '--help']
588 args, _ = parser.parse_known_args(args)
590 # Configure initial bootstrap
594 rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
595 rb.build_dir = os.path.join(os.getcwd(), "build")
596 rb.verbose = args.verbose
597 rb.clean = args.clean
600 with open(args.config or 'config.toml') as config:
601 rb.config_toml = config.read()
605 rb.config_mk = open('config.mk').read()
609 if '\nverbose = 2' in rb.config_toml:
611 elif '\nverbose = 1' in rb.config_toml:
614 rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
615 'CFG_ENABLE_VENDOR' in rb.config_mk
617 rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
618 'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
620 if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
621 if os.environ.get('USER') != os.environ['SUDO_USER']:
622 rb.use_vendored_sources = True
623 print('info: looks like you are running this command under `sudo`')
624 print(' and so in order to preserve your $HOME this will now')
625 print(' use vendored sources by default. Note that if this')
626 print(' does not work you should run a normal build first')
627 print(' before running a command like `sudo make install`')
629 if rb.use_vendored_sources:
630 if not os.path.exists('.cargo'):
631 os.makedirs('.cargo')
632 with open('.cargo/config', 'w') as f:
635 replace-with = 'vendored-sources'
636 registry = 'https://example.com'
638 [source.vendored-sources]
639 directory = '{}/src/vendor'
640 """.format(rb.rust_root))
642 if os.path.exists('.cargo'):
643 shutil.rmtree('.cargo')
645 data = stage0_data(rb.rust_root)
646 rb._date = data['date']
647 rb._rustc_channel = data['rustc']
648 rb._cargo_channel = data['cargo']
650 rb._download_url = 'https://dev-static.rust-lang.org'
652 rb._download_url = 'https://static.rust-lang.org'
654 rb.update_submodules()
656 # Fetch/build the bootstrap
657 rb.build = rb.build_triple()
664 args = [rb.bootstrap_binary()]
665 args.extend(sys.argv[1:])
666 env = os.environ.copy()
667 env["BUILD"] = rb.build
668 env["SRC"] = rb.rust_root
669 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
670 run(args, env=env, verbose=rb.verbose)
676 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
679 if not help_triggered:
680 print("Build completed successfully in %s" %
681 format_build_time(time() - start_time))
682 except (SystemExit, KeyboardInterrupt) as e:
683 if hasattr(e, 'code') and isinstance(e.code, int):
688 if not help_triggered:
689 print("Build completed unsuccessfully in %s" %
690 format_build_time(time() - start_time))
693 if __name__ == '__main__':