src/etc/snapshot.py

   1 # Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 import re
  12 import os
  13 import sys
  14 import glob
  15 import tarfile
  16 import shutil
  17 import subprocess
  18 import distutils.spawn
  19
  20 try:
  21     import hashlib
  22     sha_func = hashlib.sha1
  23 except ImportError:
  24     import sha
  25     sha_func = sha.new
  26
  27
  28 def scrub(b):
  29     if sys.version_info >= (3,) and type(b) == bytes:
  30         return b.decode('ascii')
  31     else:
  32         return b
  33
  34 src_dir = scrub(os.getenv("CFG_SRC_DIR"))
  35 if not src_dir:
  36     raise Exception("missing env var CFG_SRC_DIR")
  37
  38 snapshotfile = os.path.join(src_dir, "src", "snapshots.txt")
  39 download_url_base = "https://static.rust-lang.org/stage0-snapshots"
  40 download_dir_base = "dl"
  41 download_unpack_base = os.path.join(download_dir_base, "unpack")
  42
  43 snapshot_files = {
  44         "bitrig": ["bin/rustc"],
  45         "dragonfly": ["bin/rustc"],
  46         "freebsd": ["bin/rustc"],
  47         "linux": ["bin/rustc"],
  48         "macos": ["bin/rustc"],
  49         "netbsd": ["bin/rustc"],
  50         "openbsd": ["bin/rustc"],
  51         "winnt": ["bin/rustc.exe"],
  52         }
  53
  54 winnt_runtime_deps_32 = ["libgcc_s_dw2-1.dll", "libstdc++-6.dll"]
  55 winnt_runtime_deps_64 = ["libgcc_s_seh-1.dll", "libstdc++-6.dll"]
  56
  57 def parse_line(n, line):
  58     global snapshotfile
  59
  60     if re.match(r"\s*$", line):
  61         return None
  62
  63     if re.match(r"^T\s*$", line):
  64         return None
  65
  66     match = re.match(r"\s+([\w_-]+) ([a-fA-F\d]{40})\s*$", line)
  67     if match:
  68         return {"type": "file",
  69                 "platform": match.group(1),
  70                 "hash": match.group(2).lower()}
  71
  72     match = re.match(r"([ST]) (\d{4}-\d{2}-\d{2}) ([a-fA-F\d]+)\s*$", line)
  73     if not match:
  74         raise Exception("%s:%d:E syntax error: " % (snapshotfile, n))
  75     return {"type": "snapshot",
  76             "date": match.group(2),
  77             "rev": match.group(3)}
  78
  79
  80 def partial_snapshot_name(date, rev, platform):
  81     return ("rust-stage0-%s-%s-%s.tar.bz2" %
  82             (date, rev, platform))
  83
  84
  85 def full_snapshot_name(date, rev, platform, hsh):
  86     return ("rust-stage0-%s-%s-%s-%s.tar.bz2" %
  87             (date, rev, platform, hsh))
  88
  89
  90 def get_kernel(triple):
  91     t = triple.split('-')
  92     if len(t) == 2:
  93         os_name = t[1]
  94     else:
  95         os_name = t[2]
  96
  97     if os_name == "windows":
  98         return "winnt"
  99     if os_name == "darwin":
 100         return "macos"
 101     if os_name == "freebsd":
 102         return "freebsd"
 103     if os_name == "dragonfly":
 104         return "dragonfly"
 105     if os_name == "bitrig":
 106         return "bitrig"
 107     if os_name == "netbsd":
 108         return "netbsd"
 109     if os_name == "openbsd":
 110         return "openbsd"
 111     return "linux"
 112
 113
 114 def get_cpu(triple):
 115     arch = triple.split('-')[0]
 116     if arch == "i686":
 117         return "i386"
 118     return arch
 119
 120
 121 def get_platform(triple):
 122     return "%s-%s" % (get_kernel(triple), get_cpu(triple))
 123
 124
 125 def cmd_out(cmdline):
 126     p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
 127     return scrub(p.communicate()[0].strip())
 128
 129
 130 def local_rev_info(field):
 131     return cmd_out(["git", "--git-dir=" + os.path.join(src_dir, ".git"),
 132                     "log", "-n", "1",
 133                     "--format=%%%s" % field, "HEAD"])
 134
 135
 136 def local_rev_full_sha():
 137     return local_rev_info("H").split()[0]
 138
 139
 140 def local_rev_short_sha():
 141     return local_rev_info("h").split()[0]
 142
 143
 144 def local_rev_committer_date():
 145     return local_rev_info("ci")
 146
 147
 148 def get_url_to_file(u, f):
 149     # no security issue, just to stop partial download leaving a stale file
 150     tmpf = f + '.tmp'
 151
 152     returncode = -1
 153     if distutils.spawn.find_executable("curl"):
 154         returncode = subprocess.call(["curl", "-o", tmpf, u])
 155     elif distutils.spawn.find_executable("wget"):
 156         returncode = subprocess.call(["wget", "-O", tmpf, u])
 157
 158     if returncode != 0:
 159         try:
 160             os.unlink(tmpf)
 161         except OSError:
 162             pass
 163         raise Exception("failed to fetch url")
 164     os.rename(tmpf, f)
 165
 166
 167 def snap_filename_hash_part(snap):
 168     match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
 169     if not match:
 170         raise Exception("unable to find hash in filename: " + snap)
 171     return match.group(1)
 172
 173
 174 def hash_file(x):
 175     h = sha_func()
 176     h.update(open(x, "rb").read())
 177     return scrub(h.hexdigest())
 178
 179
 180 def get_winnt_runtime_deps(platform):
 181     """Returns a list of paths of Rust's system runtime dependencies"""
 182     if platform == "winnt-x86_64":
 183         deps = winnt_runtime_deps_64
 184     else:
 185         deps = winnt_runtime_deps_32
 186     runtime_deps = []
 187     path_dirs = os.environ["PATH"].split(os.pathsep)
 188     for name in deps:
 189         for dir in path_dirs:
 190             filepath = os.path.join(dir, name)
 191             if os.path.isfile(filepath):
 192                 runtime_deps.append(filepath)
 193                 break
 194         else:
 195             raise Exception("Could not find runtime dependency: %s" % name)
 196     return runtime_deps
 197
 198
 199 def make_snapshot(stage, triple):
 200     kernel = get_kernel(triple)
 201     platform = get_platform(triple)
 202     rev = local_rev_short_sha()
 203     date = local_rev_committer_date().split()[0]
 204
 205     file0 = partial_snapshot_name(date, rev, platform)
 206
 207     def in_tar_name(fn):
 208         cs = re.split(r"[\\/]", fn)
 209         if len(cs) >= 2:
 210             return os.sep.join(cs[-2:])
 211
 212     tar = tarfile.open(file0, "w:bz2")
 213
 214     for name in snapshot_files[kernel]:
 215         dir = stage
 216         if stage == "stage1" and re.match(r"^lib/(lib)?std.*", name):
 217             dir = "stage0"
 218         fn_glob = os.path.join(triple, dir, name)
 219         matches = glob.glob(fn_glob)
 220         if not matches:
 221             raise Exception("Not found file with name like " + fn_glob)
 222         if len(matches) == 1:
 223             tar.add(matches[0], "rust-stage0/" + in_tar_name(matches[0]))
 224         else:
 225             raise Exception("Found stale files: \n  %s\n"
 226                             "Please make a clean build." % "\n  ".join(matches))
 227
 228     if kernel == "winnt":
 229         for path in get_winnt_runtime_deps(platform):
 230             tar.add(path, "rust-stage0/bin/" + os.path.basename(path))
 231         tar.add(os.path.join(os.path.dirname(__file__), "third-party"),
 232                 "rust-stage0/bin/third-party")
 233
 234     tar.close()
 235
 236     h = hash_file(file0)
 237     file1 = full_snapshot_name(date, rev, platform, h)
 238
 239     shutil.move(file0, file1)
 240
 241     return file1
 242
 243
 244 def curr_snapshot_rev():
 245     i = 0
 246     found_snap = False
 247     date = None
 248     rev = None
 249
 250     f = open(snapshotfile)
 251     for line in f.readlines():
 252         i += 1
 253         parsed = parse_line(i, line)
 254         if not parsed:
 255             continue
 256
 257         if parsed["type"] == "snapshot":
 258             date = parsed["date"]
 259             rev = parsed["rev"]
 260             found_snap = True
 261             break
 262
 263     if not found_snap:
 264         raise Exception("no snapshot entries in file")
 265
 266     return (date, rev)
 267
 268
 269 def determine_curr_snapshot(triple):
 270     i = 0
 271     platform = get_platform(triple)
 272
 273     found_file = False
 274     found_snap = False
 275     hsh = None
 276     date = None
 277     rev = None
 278
 279     f = open(snapshotfile)
 280     for line in f.readlines():
 281         i += 1
 282         parsed = parse_line(i, line)
 283         if not parsed:
 284             continue
 285
 286         if found_snap and parsed["type"] == "file":
 287             if parsed["platform"] == platform:
 288                 hsh = parsed["hash"]
 289                 found_file = True
 290                 break
 291         elif parsed["type"] == "snapshot":
 292             date = parsed["date"]
 293             rev = parsed["rev"]
 294             found_snap = True
 295
 296     if not found_snap:
 297         raise Exception("no snapshot entries in file")
 298
 299     if not found_file:
 300         raise Exception("no snapshot file found for platform %s, rev %s" %
 301                         (platform, rev))
 302
 303     return full_snapshot_name(date, rev, platform, hsh)