import datetime
import distutils.version
import hashlib
+import json
import os
import re
import shutil
except tarfile.CompressionError:
return False
-def get(url, path, verbose=False, do_verify=True):
- suffix = '.sha256'
- sha_url = url + suffix
+def get(base, url, path, checksums, verbose=False, do_verify=True):
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
- with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
- sha_path = sha_file.name
try:
if do_verify:
- download(sha_path, sha_url, False, verbose)
+ if url not in checksums:
+ raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
+ sha256 = checksums[url]
if os.path.exists(path):
- if verify(path, sha_path, False):
+ if verify(path, sha256, False):
if verbose:
print("using already-download file", path)
return
print("ignoring already-download file",
path, "due to failed verification")
os.unlink(path)
- download(temp_path, url, True, verbose)
- if do_verify and not verify(temp_path, sha_path, verbose):
+ download(temp_path, "{}/{}".format(base, url), True, verbose)
+ if do_verify and not verify(temp_path, sha256, verbose):
raise RuntimeError("failed verification")
if verbose:
print("moving {} to {}".format(temp_path, path))
shutil.move(temp_path, path)
finally:
- delete_if_present(sha_path, verbose)
- delete_if_present(temp_path, verbose)
-
-
-def delete_if_present(path, verbose):
- """Remove the given file if present"""
- if os.path.isfile(path):
- if verbose:
- print("removing", path)
- os.unlink(path)
+ if os.path.isfile(temp_path):
+ if verbose:
+ print("removing", temp_path)
+ os.unlink(temp_path)
def download(path, url, probably_big, verbose):
exception=exception)
-def verify(path, sha_path, verbose):
+def verify(path, expected, verbose):
"""Check if the sha256 sum of the given path is valid"""
if verbose:
print("verifying", path)
with open(path, "rb") as source:
found = hashlib.sha256(source.read()).hexdigest()
- with open(sha_path, "r") as sha256sum:
- expected = sha256sum.readline().split()[0]
verified = found == expected
if not verified:
print("invalid checksum:\n"
sys.exit(1)
-def stage0_data(rust_root):
- """Build a dictionary from stage0.txt"""
- nightlies = os.path.join(rust_root, "src/stage0.txt")
- with open(nightlies, 'r') as nightlies:
- lines = [line.rstrip() for line in nightlies
- if not line.startswith("#")]
- return dict([line.split(": ", 1) for line in lines if line])
-
-
def format_build_time(duration):
"""Return a nicer format for build time
'i486': 'i686',
'i686': 'i686',
'i786': 'i686',
+ 'm68k': 'm68k',
'powerpc': 'powerpc',
'powerpc64': 'powerpc64',
'powerpc64le': 'powerpc64le',
'ppc': 'powerpc',
'ppc64': 'powerpc64',
'ppc64le': 'powerpc64le',
+ 'riscv64': 'riscv64gc',
's390x': 's390x',
'x64': 'x86_64',
'x86': 'i686',
os.rename(tmp, filepath)
+class Stage0Toolchain:
+ def __init__(self, stage0_payload):
+ self.date = stage0_payload["date"]
+ self.version = stage0_payload["version"]
+
+ def channel(self):
+ return self.version + "-" + self.date
+
+
class RustBuild(object):
"""Provide all the methods required to build Rust"""
def __init__(self):
- self.date = ''
+ self.checksums_sha256 = {}
+ self.stage0_compiler = None
+ self.stage0_rustfmt = None
self._download_url = ''
- self.rustc_channel = ''
- self.rustfmt_channel = ''
self.build = ''
self.build_dir = ''
self.clean = False
will move all the content to the right place.
"""
if rustc_channel is None:
- rustc_channel = self.rustc_channel
- rustfmt_channel = self.rustfmt_channel
+ rustc_channel = self.stage0_compiler.version
bin_root = self.bin_root(stage0)
- key = self.date
+ key = self.stage0_compiler.date
if not stage0:
key += str(self.rustc_commit)
if self.rustc(stage0).startswith(bin_root) and \
if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
not os.path.exists(self.rustfmt())
- or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
+ or self.program_out_of_date(
+ self.rustfmt_stamp(),
+ "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
+ )
):
- if rustfmt_channel:
+ if self.stage0_rustfmt is not None:
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
- [channel, date] = rustfmt_channel.split('-', 1)
- filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
+ filename = "rustfmt-{}-{}{}".format(
+ self.stage0_rustfmt.version, self.build, tarball_suffix,
+ )
self._download_component_helper(
- filename, "rustfmt-preview", tarball_suffix, key=date
+ filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
)
self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
with output(self.rustfmt_stamp()) as rustfmt_stamp:
- rustfmt_stamp.write(self.rustfmt_channel)
+ rustfmt_stamp.write(self.stage0_rustfmt.channel())
# Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
if self.downloading_llvm() and stage0:
):
if key is None:
if stage0:
- key = self.date
+ key = self.stage0_compiler.date
else:
key = self.rustc_commit
cache_dst = os.path.join(self.build_dir, "cache")
os.makedirs(rustc_cache)
if stage0:
- url = "{}/dist/{}".format(self._download_url, key)
+ base = self._download_url
+ url = "dist/{}".format(key)
else:
- url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
+ base = "https://ci-artifacts.rust-lang.org"
+ url = "rustc-builds/{}".format(self.rustc_commit)
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
- get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0)
+ get(
+ base,
+ "{}/{}".format(url, filename),
+ tarball,
+ self.checksums_sha256,
+ verbose=self.verbose,
+ do_verify=stage0,
+ )
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
def _download_ci_llvm(self, llvm_sha, llvm_assertions):
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
- url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
+ base = "https://ci-artifacts.rust-lang.org"
+ url = "rustc-builds/{}".format(llvm_sha)
if llvm_assertions:
url = url.replace('rustc-builds', 'rustc-builds-alt')
# ci-artifacts are only stored as .xz, not .gz
filename = "rust-dev-nightly-" + self.build + tarball_suffix
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
- get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
+ get(
+ base,
+ "{}/{}".format(url, filename),
+ tarball,
+ self.checksums_sha256,
+ verbose=self.verbose,
+ do_verify=False,
+ )
unpack(tarball, tarball_suffix, self.llvm_root(),
match="rust-dev",
verbose=self.verbose)
if ostype != "Linux":
return
- # Use `/etc/os-release` instead of `/etc/NIXOS`.
- # The latter one does not exist on NixOS when using tmpfs as root.
- try:
- with open("/etc/os-release", "r") as f:
- if not any(line.strip() == "ID=nixos" for line in f):
- return
- except FileNotFoundError:
- return
- if os.path.exists("/lib"):
- return
+ # If the user has asked binaries to be patched for Nix, then
+ # don't check for NixOS or `/lib`, just continue to the patching.
+ if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
+ # Use `/etc/os-release` instead of `/etc/NIXOS`.
+ # The latter one does not exist on NixOS when using tmpfs as root.
+ try:
+ with open("/etc/os-release", "r") as f:
+ if not any(line.strip() == "ID=nixos" for line in f):
+ return
+ except FileNotFoundError:
+ return
+ if os.path.exists("/lib"):
+ return
- # At this point we're pretty sure the user is running NixOS
- nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
+ # At this point we're pretty sure the user is running NixOS or
+ # using Nix
+ nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
print(nix_os_msg, fname)
# Only build `.nix-deps` once.
def rustfmt(self):
"""Return config path for rustfmt"""
- if not self.rustfmt_channel:
+ if self.stage0_rustfmt is None:
return None
return self.program_config('rustfmt')
self.update_submodule(module[0], module[1], recorded_submodules)
print("Submodules updated in %.2f seconds" % (time() - start_time))
- def set_normal_environment(self):
+ def set_dist_environment(self, url):
"""Set download URL for normal environment"""
if 'RUSTUP_DIST_SERVER' in os.environ:
self._download_url = os.environ['RUSTUP_DIST_SERVER']
else:
- self._download_url = 'https://static.rust-lang.org'
-
- def set_dev_environment(self):
- """Set download URL for development environment"""
- if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
- self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
- else:
- self._download_url = 'https://dev-static.rust-lang.org'
+ self._download_url = url
def check_vendored_status(self):
"""Check that vendoring is configured properly"""
build_dir = build.get_toml('build-dir', 'build') or 'build'
build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
- data = stage0_data(build.rust_root)
- build.date = data['date']
- build.rustc_channel = data['rustc']
+ with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
+ data = json.load(f)
+ build.checksums_sha256 = data["checksums_sha256"]
+ build.stage0_compiler = Stage0Toolchain(data["compiler"])
+ if data.get("rustfmt") is not None:
+ build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
- if "rustfmt" in data:
- build.rustfmt_channel = data['rustfmt']
-
- if 'dev' in data:
- build.set_dev_environment()
- else:
- build.set_normal_environment()
+ build.set_dist_environment(data["dist_server"])
build.build = args.build or build.build_triple()
build.update_submodules()
projects / rust.git / blobdiff