src/etc/snapshot.py

   1 # Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 import re
  12 import os
  13 import sys
  14 import glob
  15 import tarfile
  16 import shutil
  17 import subprocess
  18 import distutils.spawn
  19
  20 try:
  21     import hashlib
  22     sha_func = hashlib.sha1
  23 except ImportError:
  24     import sha
  25     sha_func = sha.new
  26
  27
  28 def scrub(b):
  29     if sys.version_info >= (3,) and type(b) == bytes:
  30         return b.decode('ascii')
  31     else:
  32         return b
  33
  34 src_dir = scrub(os.getenv("CFG_SRC_DIR"))
  35 if not src_dir:
  36     raise Exception("missing env var CFG_SRC_DIR")
  37
  38 snapshotfile = os.path.join(src_dir, "src", "snapshots.txt")
  39 download_url_base = "https://static.rust-lang.org/stage0-snapshots"
  40 download_dir_base = "dl"
  41 download_unpack_base = os.path.join(download_dir_base, "unpack")
  42
  43 snapshot_files = {
  44         "linux": ["bin/rustc"],
  45         "macos": ["bin/rustc"],
  46         "winnt": ["bin/rustc.exe"],
  47         "freebsd": ["bin/rustc"],
  48         "dragonfly": ["bin/rustc"],
  49         "openbsd": ["bin/rustc"],
  50         }
  51
  52 winnt_runtime_deps_32 = ["libgcc_s_dw2-1.dll", "libstdc++-6.dll"]
  53 winnt_runtime_deps_64 = ["libgcc_s_seh-1.dll", "libstdc++-6.dll"]
  54
  55
  56 def parse_line(n, line):
  57     global snapshotfile
  58
  59     if re.match(r"\s*$", line):
  60         return None
  61
  62     if re.match(r"^T\s*$", line):
  63         return None
  64
  65     match = re.match(r"\s+([\w_-]+) ([a-fA-F\d]{40})\s*$", line)
  66     if match:
  67         return {"type": "file",
  68                 "platform": match.group(1),
  69                 "hash": match.group(2).lower()}
  70
  71     match = re.match(r"([ST]) (\d{4}-\d{2}-\d{2}) ([a-fA-F\d]+)\s*$", line)
  72     if not match:
  73         raise Exception("%s:%d:E syntax error: " % (snapshotfile, n))
  74     return {"type": "snapshot",
  75             "date": match.group(2),
  76             "rev": match.group(3)}
  77
  78
  79 def partial_snapshot_name(date, rev, platform):
  80     return ("rust-stage0-%s-%s-%s.tar.bz2" %
  81             (date, rev, platform))
  82
  83
  84 def full_snapshot_name(date, rev, platform, hsh):
  85     return ("rust-stage0-%s-%s-%s-%s.tar.bz2" %
  86             (date, rev, platform, hsh))
  87
  88
  89 def get_kernel(triple):
  90     t = triple.split('-')
  91     if len(t) == 2:
  92         os_name = t[1]
  93     else:
  94         os_name = t[2]
  95
  96     if os_name == "windows":
  97         return "winnt"
  98     if os_name == "darwin":
  99         return "macos"
 100     if os_name == "freebsd":
 101         return "freebsd"
 102     if os_name == "dragonfly":
 103         return "dragonfly"
 104     if os_name == "openbsd":
 105         return "openbsd"
 106     return "linux"
 107
 108
 109 def get_cpu(triple):
 110     arch = triple.split('-')[0]
 111     if arch == "i686":
 112         return "i386"
 113     return arch
 114
 115
 116 def get_platform(triple):
 117     return "%s-%s" % (get_kernel(triple), get_cpu(triple))
 118
 119
 120 def cmd_out(cmdline):
 121     p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
 122     return scrub(p.communicate()[0].strip())
 123
 124
 125 def local_rev_info(field):
 126     return cmd_out(["git", "--git-dir=" + os.path.join(src_dir, ".git"),
 127                     "log", "-n", "1",
 128                     "--format=%%%s" % field, "HEAD"])
 129
 130
 131 def local_rev_full_sha():
 132     return local_rev_info("H").split()[0]
 133
 134
 135 def local_rev_short_sha():
 136     return local_rev_info("h").split()[0]
 137
 138
 139 def local_rev_committer_date():
 140     return local_rev_info("ci")
 141
 142
 143 def get_url_to_file(u, f):
 144     # no security issue, just to stop partial download leaving a stale file
 145     tmpf = f + '.tmp'
 146
 147     returncode = -1
 148     if distutils.spawn.find_executable("curl"):
 149         returncode = subprocess.call(["curl", "-o", tmpf, u])
 150     elif distutils.spawn.find_executable("wget"):
 151         returncode = subprocess.call(["wget", "-O", tmpf, u])
 152
 153     if returncode != 0:
 154         try:
 155             os.unlink(tmpf)
 156         except OSError:
 157             pass
 158         raise Exception("failed to fetch url")
 159     os.rename(tmpf, f)
 160
 161
 162 def snap_filename_hash_part(snap):
 163     match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
 164     if not match:
 165         raise Exception("unable to find hash in filename: " + snap)
 166     return match.group(1)
 167
 168
 169 def hash_file(x):
 170     h = sha_func()
 171     h.update(open(x, "rb").read())
 172     return scrub(h.hexdigest())
 173
 174
 175 def get_winnt_runtime_deps(platform):
 176     """Returns a list of paths of Rust's system runtime dependencies"""
 177     if platform == "winnt-x86_64":
 178         deps = winnt_runtime_deps_64
 179     else:
 180         deps = winnt_runtime_deps_32
 181     runtime_deps = []
 182     path_dirs = os.environ["PATH"].split(os.pathsep)
 183     for name in deps:
 184         for dir in path_dirs:
 185             filepath = os.path.join(dir, name)
 186             if os.path.isfile(filepath):
 187                 runtime_deps.append(filepath)
 188                 break
 189         else:
 190             raise Exception("Could not find runtime dependency: %s" % name)
 191     return runtime_deps
 192
 193
 194 def make_snapshot(stage, triple):
 195     kernel = get_kernel(triple)
 196     platform = get_platform(triple)
 197     rev = local_rev_short_sha()
 198     date = local_rev_committer_date().split()[0]
 199
 200     file0 = partial_snapshot_name(date, rev, platform)
 201
 202     def in_tar_name(fn):
 203         cs = re.split(r"[\\/]", fn)
 204         if len(cs) >= 2:
 205             return os.sep.join(cs[-2:])
 206
 207     tar = tarfile.open(file0, "w:bz2")
 208
 209     for name in snapshot_files[kernel]:
 210         dir = stage
 211         if stage == "stage1" and re.match(r"^lib/(lib)?std.*", name):
 212             dir = "stage0"
 213         fn_glob = os.path.join(triple, dir, name)
 214         matches = glob.glob(fn_glob)
 215         if not matches:
 216             raise Exception("Not found file with name like " + fn_glob)
 217         if len(matches) == 1:
 218             tar.add(matches[0], "rust-stage0/" + in_tar_name(matches[0]))
 219         else:
 220             raise Exception("Found stale files: \n  %s\n"
 221                             "Please make a clean build." % "\n  ".join(matches))
 222
 223     if kernel == "winnt":
 224         for path in get_winnt_runtime_deps(platform):
 225             tar.add(path, "rust-stage0/bin/" + os.path.basename(path))
 226         tar.add(os.path.join(os.path.dirname(__file__), "third-party"),
 227                 "rust-stage0/bin/third-party")
 228
 229     tar.close()
 230
 231     h = hash_file(file0)
 232     file1 = full_snapshot_name(date, rev, platform, h)
 233
 234     shutil.move(file0, file1)
 235
 236     return file1
 237
 238
 239 def curr_snapshot_rev():
 240     i = 0
 241     found_snap = False
 242     date = None
 243     rev = None
 244
 245     f = open(snapshotfile)
 246     for line in f.readlines():
 247         i += 1
 248         parsed = parse_line(i, line)
 249         if not parsed:
 250             continue
 251
 252         if parsed["type"] == "snapshot":
 253             date = parsed["date"]
 254             rev = parsed["rev"]
 255             found_snap = True
 256             break
 257
 258     if not found_snap:
 259         raise Exception("no snapshot entries in file")
 260
 261     return (date, rev)
 262
 263
 264 def determine_curr_snapshot(triple):
 265     i = 0
 266     platform = get_platform(triple)
 267
 268     found_file = False
 269     found_snap = False
 270     hsh = None
 271     date = None
 272     rev = None
 273
 274     f = open(snapshotfile)
 275     for line in f.readlines():
 276         i += 1
 277         parsed = parse_line(i, line)
 278         if not parsed:
 279             continue
 280
 281         if found_snap and parsed["type"] == "file":
 282             if parsed["platform"] == platform:
 283                 hsh = parsed["hash"]
 284                 found_file = True
 285                 break
 286         elif parsed["type"] == "snapshot":
 287             date = parsed["date"]
 288             rev = parsed["rev"]
 289             found_snap = True
 290
 291     if not found_snap:
 292         raise Exception("no snapshot entries in file")
 293
 294     if not found_file:
 295         raise Exception("no snapshot file found for platform %s, rev %s" %
 296                         (platform, rev))
 297
 298     return full_snapshot_name(date, rev, platform, hsh)