src/etc/snapshot.py

   1 # Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 import re
  12 import os
  13 import sys
  14 import glob
  15 import tarfile
  16 import shutil
  17 import subprocess
  18 import distutils.spawn
  19
  20 try:
  21     import hashlib
  22     sha_func = hashlib.sha1
  23 except ImportError:
  24     import sha
  25     sha_func = sha.new
  26
  27
  28 def scrub(b):
  29     if sys.version_info >= (3,) and type(b) == bytes:
  30         return b.decode('ascii')
  31     else:
  32         return b
  33
  34 src_dir = scrub(os.getenv("CFG_SRC_DIR"))
  35 if not src_dir:
  36     raise Exception("missing env var CFG_SRC_DIR")
  37
  38 snapshotfile = os.path.join(src_dir, "src", "snapshots.txt")
  39 download_url_base = "https://static.rust-lang.org/stage0-snapshots"
  40 download_dir_base = "dl"
  41 download_unpack_base = os.path.join(download_dir_base, "unpack")
  42
  43 snapshot_files = {
  44         "linux": ["bin/rustc"],
  45         "macos": ["bin/rustc"],
  46         "winnt": ["bin/rustc.exe"],
  47         "freebsd": ["bin/rustc"],
  48         "dragonfly": ["bin/rustc"],
  49         "bitrig": ["bin/rustc"],
  50         "openbsd": ["bin/rustc"],
  51         }
  52
  53 winnt_runtime_deps_32 = ["libgcc_s_dw2-1.dll", "libstdc++-6.dll"]
  54 winnt_runtime_deps_64 = ["libgcc_s_seh-1.dll", "libstdc++-6.dll"]
  55
  56 def parse_line(n, line):
  57     global snapshotfile
  58
  59     if re.match(r"\s*$", line):
  60         return None
  61
  62     if re.match(r"^T\s*$", line):
  63         return None
  64
  65     match = re.match(r"\s+([\w_-]+) ([a-fA-F\d]{40})\s*$", line)
  66     if match:
  67         return {"type": "file",
  68                 "platform": match.group(1),
  69                 "hash": match.group(2).lower()}
  70
  71     match = re.match(r"([ST]) (\d{4}-\d{2}-\d{2}) ([a-fA-F\d]+)\s*$", line)
  72     if not match:
  73         raise Exception("%s:%d:E syntax error: " % (snapshotfile, n))
  74     return {"type": "snapshot",
  75             "date": match.group(2),
  76             "rev": match.group(3)}
  77
  78
  79 def partial_snapshot_name(date, rev, platform):
  80     return ("rust-stage0-%s-%s-%s.tar.bz2" %
  81             (date, rev, platform))
  82
  83
  84 def full_snapshot_name(date, rev, platform, hsh):
  85     return ("rust-stage0-%s-%s-%s-%s.tar.bz2" %
  86             (date, rev, platform, hsh))
  87
  88
  89 def get_kernel(triple):
  90     t = triple.split('-')
  91     if len(t) == 2:
  92         os_name = t[1]
  93     else:
  94         os_name = t[2]
  95
  96     if os_name == "windows":
  97         return "winnt"
  98     if os_name == "darwin":
  99         return "macos"
 100     if os_name == "freebsd":
 101         return "freebsd"
 102     if os_name == "dragonfly":
 103         return "dragonfly"
 104     if os_name == "bitrig":
 105         return "bitrig"
 106     if os_name == "openbsd":
 107         return "openbsd"
 108     return "linux"
 109
 110
 111 def get_cpu(triple):
 112     arch = triple.split('-')[0]
 113     if arch == "i686":
 114         return "i386"
 115     return arch
 116
 117
 118 def get_platform(triple):
 119     return "%s-%s" % (get_kernel(triple), get_cpu(triple))
 120
 121
 122 def cmd_out(cmdline):
 123     p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
 124     return scrub(p.communicate()[0].strip())
 125
 126
 127 def local_rev_info(field):
 128     return cmd_out(["git", "--git-dir=" + os.path.join(src_dir, ".git"),
 129                     "log", "-n", "1",
 130                     "--format=%%%s" % field, "HEAD"])
 131
 132
 133 def local_rev_full_sha():
 134     return local_rev_info("H").split()[0]
 135
 136
 137 def local_rev_short_sha():
 138     return local_rev_info("h").split()[0]
 139
 140
 141 def local_rev_committer_date():
 142     return local_rev_info("ci")
 143
 144
 145 def get_url_to_file(u, f):
 146     # no security issue, just to stop partial download leaving a stale file
 147     tmpf = f + '.tmp'
 148
 149     returncode = -1
 150     if distutils.spawn.find_executable("curl"):
 151         returncode = subprocess.call(["curl", "-o", tmpf, u])
 152     elif distutils.spawn.find_executable("wget"):
 153         returncode = subprocess.call(["wget", "-O", tmpf, u])
 154
 155     if returncode != 0:
 156         try:
 157             os.unlink(tmpf)
 158         except OSError:
 159             pass
 160         raise Exception("failed to fetch url")
 161     os.rename(tmpf, f)
 162
 163
 164 def snap_filename_hash_part(snap):
 165     match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
 166     if not match:
 167         raise Exception("unable to find hash in filename: " + snap)
 168     return match.group(1)
 169
 170
 171 def hash_file(x):
 172     h = sha_func()
 173     h.update(open(x, "rb").read())
 174     return scrub(h.hexdigest())
 175
 176
 177 def get_winnt_runtime_deps(platform):
 178     """Returns a list of paths of Rust's system runtime dependencies"""
 179     if platform == "winnt-x86_64":
 180         deps = winnt_runtime_deps_64
 181     else:
 182         deps = winnt_runtime_deps_32
 183     runtime_deps = []
 184     path_dirs = os.environ["PATH"].split(os.pathsep)
 185     for name in deps:
 186         for dir in path_dirs:
 187             filepath = os.path.join(dir, name)
 188             if os.path.isfile(filepath):
 189                 runtime_deps.append(filepath)
 190                 break
 191         else:
 192             raise Exception("Could not find runtime dependency: %s" % name)
 193     return runtime_deps
 194
 195
 196 def make_snapshot(stage, triple):
 197     kernel = get_kernel(triple)
 198     platform = get_platform(triple)
 199     rev = local_rev_short_sha()
 200     date = local_rev_committer_date().split()[0]
 201
 202     file0 = partial_snapshot_name(date, rev, platform)
 203
 204     def in_tar_name(fn):
 205         cs = re.split(r"[\\/]", fn)
 206         if len(cs) >= 2:
 207             return os.sep.join(cs[-2:])
 208
 209     tar = tarfile.open(file0, "w:bz2")
 210
 211     for name in snapshot_files[kernel]:
 212         dir = stage
 213         if stage == "stage1" and re.match(r"^lib/(lib)?std.*", name):
 214             dir = "stage0"
 215         fn_glob = os.path.join(triple, dir, name)
 216         matches = glob.glob(fn_glob)
 217         if not matches:
 218             raise Exception("Not found file with name like " + fn_glob)
 219         if len(matches) == 1:
 220             tar.add(matches[0], "rust-stage0/" + in_tar_name(matches[0]))
 221         else:
 222             raise Exception("Found stale files: \n  %s\n"
 223                             "Please make a clean build." % "\n  ".join(matches))
 224
 225     if kernel == "winnt":
 226         for path in get_winnt_runtime_deps(platform):
 227             tar.add(path, "rust-stage0/bin/" + os.path.basename(path))
 228         tar.add(os.path.join(os.path.dirname(__file__), "third-party"),
 229                 "rust-stage0/bin/third-party")
 230
 231     tar.close()
 232
 233     h = hash_file(file0)
 234     file1 = full_snapshot_name(date, rev, platform, h)
 235
 236     shutil.move(file0, file1)
 237
 238     return file1
 239
 240
 241 def curr_snapshot_rev():
 242     i = 0
 243     found_snap = False
 244     date = None
 245     rev = None
 246
 247     f = open(snapshotfile)
 248     for line in f.readlines():
 249         i += 1
 250         parsed = parse_line(i, line)
 251         if not parsed:
 252             continue
 253
 254         if parsed["type"] == "snapshot":
 255             date = parsed["date"]
 256             rev = parsed["rev"]
 257             found_snap = True
 258             break
 259
 260     if not found_snap:
 261         raise Exception("no snapshot entries in file")
 262
 263     return (date, rev)
 264
 265
 266 def determine_curr_snapshot(triple):
 267     i = 0
 268     platform = get_platform(triple)
 269
 270     found_file = False
 271     found_snap = False
 272     hsh = None
 273     date = None
 274     rev = None
 275
 276     f = open(snapshotfile)
 277     for line in f.readlines():
 278         i += 1
 279         parsed = parse_line(i, line)
 280         if not parsed:
 281             continue
 282
 283         if found_snap and parsed["type"] == "file":
 284             if parsed["platform"] == platform:
 285                 hsh = parsed["hash"]
 286                 found_file = True
 287                 break
 288         elif parsed["type"] == "snapshot":
 289             date = parsed["date"]
 290             rev = parsed["rev"]
 291             found_snap = True
 292
 293     if not found_snap:
 294         raise Exception("no snapshot entries in file")
 295
 296     if not found_file:
 297         raise Exception("no snapshot file found for platform %s, rev %s" %
 298                         (platform, rev))
 299
 300     return full_snapshot_name(date, rev, platform, hsh)