Initial sketch of snapshot support code, based on code from marijn.

src/etc/get-snapshot.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+import os, tarfile, hashlib, re, shutil
+from snapshot import *
+
+def snap_filename_hash_part(snap):
+  match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
+  if not match:
+    raise Exception("unable to find hash in filename: " + snap)
+  return match.group(1)
+
+def get_snapshot_and_check_hash(snap):
+
+  hsh = snap_filename_hash_part(snap)
+
+  h = hashlib.sha1()
+  url = download_url_base + "/" + snap
+  print "downloading " + url
+  u = urllib2.urlopen(url)
+  print "checking hash on download"
+  data = u.read()
+  h.update(data)
+  if h.hexdigest() != hsh:
+    raise Exception("hash check failed on " + snap)
+
+  print "hash ok"
+  with open(os.path.join(download_dir_base, snap), "w+b") as f:
+    f.write(data)
+  return True
+
+def unpack_snapshot(snap):
+  dl_path = os.path.join(download_dir_base, snap)
+  print "opening snapshot " + dl_path
+  tar = tarfile.open(dl_path)
+  kernel = get_kernel()
+  for name in snapshot_files[kernel]:
+    p = os.path.join("rust-stage0", name)
+    fp = os.path.join("stage0", name)
+    print "extracting " + fp
+    tar.extract(p, download_unpack_base)
+    tp = os.path.join(download_unpack_base, p)
+    shutil.move(tp, fp)
+  tar.close()
+  shutil.rmtree(download_unpack_base)
+
+def determine_last_snapshot_for_platform():
+  lines = open(snapshotfile).readlines();
+
+  platform = get_platform()
+
+  found = False
+  hsh = None
+  date = None
+  rev = None
+
+  for ln in range(len(lines) - 1, -1, -1):
+    parsed = parse_line(ln, lines[ln])
+    if (not parsed): continue
+
+    if parsed["type"] == "file":
+      if parsed["platform"] == platform:
+        hsh = parsed["hash"]
+    elif parsed["type"] == "snapshot":
+      date = parsed["date"]
+      rev = parsed["rev"]
+      found = True
+      break
+    elif parsed["type"] == "transition" and not foundSnapshot:
+      raise Exception("working on a transition, not updating stage0")
+
+  if not found:
+    raise Exception("no snapshot entries in file")
+
+  if not hsh:
+    raise Exception("no snapshot file found for platform %s, rev %s" %
+                    (platform, rev))
+
+  return full_snapshot_name(date, rev, get_kernel(), get_cpu(), hsh)
+
+# Main
+
+snap = determine_last_snapshot_for_platform()
+print "determined most recent snapshot: " + snap
+dl = os.path.join(download_dir_base, snap)
+if (os.path.exists(dl)):
+  if (snap_filename_hash_part(snap) == hash_file(dl)):
+    print "found existing download with ok hash"
+  else:
+    print "bad hash on existing download, re-fetching"
+    get_snapshot_and_check_hash(snap)
+else:
+  print "no cached download, fetching"
+  get_snapshot_and_check_hash(snap)
+
+unpack_snapshot(snap)

src/etc/make-snapshot.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+import shutil, tarfile
+from snapshot import *
+
+kernel = get_kernel()
+cpu = get_cpu()
+rev = local_rev_short_sha()
+date = local_rev_committer_date().split()[0]
+
+file0 = partial_snapshot_name(date, rev, kernel, cpu)
+
+tar = tarfile.open(file0, "w:bz2")
+for name in snapshot_files[kernel]:
+    tar.add(os.path.join("stage2", name),
+            os.path.join("rust-stage0", name))
+tar.close()
+
+h = hash_file(file0)
+file1 = full_snapshot_name(date, rev, kernel, cpu, h)
+
+shutil.move(file0, file1)
+
+print file1

src/etc/snapshot.py

@@ -0,0 +1,89 @@
+import re, os, sys, hashlib, tarfile, shutil, subprocess, urllib2, tempfile
+
+snapshotfile = "snapshots.txt"
+download_url_base = "http://dl.rust-lang.org/stage0-snapshots"
+download_dir_base = "dl"
+download_unpack_base = os.path.join(download_dir_base, "unpack")
+
+snapshot_files = {
+    "linux": ["rustc", "glue.o", "libstd.so" ],
+    "macos": ["rustc", "glue.o", "libstd.dylib" ],
+    "winnt": ["rustc.exe", "glue.o", "std.dll" ]
+    }
+
+def parse_line(n, line):
+  global snapshotfile
+
+  if re.match(r"\s*$", line): return None
+
+  match = re.match(r"\s+([\w_-]+) ([a-fA-F\d]{40})\s*$", line)
+  if match:
+    return { "type": "file",
+             "platform": match.group(1),
+             "hash": match.group(2).lower() }
+
+  match = re.match(r"([ST]) (\d{4}-\d{2}-\d{2}) ([a-fA-F\d]+)\s*$", line);
+  if (not match):
+    raise Exception("%s:%d:E syntax error" % (snapshotfile, n))
+  ttype = "snapshot"
+  if (match.group(1) == "T"):
+    ttype = "transition"
+  return {"type": ttype,
+          "date": match.group(2),
+          "rev": match.group(3)}
+
+
+def partial_snapshot_name(date, rev, kernel, cpu):
+  return ("rust-stage0-%s-%s-%s-%s.tar.bz2"
+          % (date, rev, kernel, cpu))
+
+def full_snapshot_name(date, rev, kernel, cpu, hsh):
+  return ("rust-stage0-%s-%s-%s-%s-%s.tar.bz2"
+          % (date, rev, kernel, cpu, hsh))
+
+
+def get_kernel():
+    if os.name == "nt":
+        return "winnt"
+    kernel = os.uname()[0].lower()
+    if kernel == "darwin":
+        kernel = "macos"
+    return kernel
+
+
+def get_cpu():
+    # return os.uname()[-1].lower()
+    return "i386"
+
+
+def get_platform():
+  return "%s-%s" % (get_kernel(), get_cpu())
+
+
+def cmd_out(cmdline):
+    p = subprocess.Popen(cmdline,
+                         stdout=subprocess.PIPE)
+    return p.communicate()[0].strip()
+
+
+def local_rev_info(field):
+    return cmd_out(["git", "log", "-n", "1",
+                    "--format=%%%s" % field, "HEAD"])
+
+
+def local_rev_full_sha():
+    return local_rev_info("H").split()[0]
+
+
+def local_rev_short_sha():
+    return local_rev_info("h").split()[0]
+
+
+def local_rev_committer_date():
+    return local_rev_info("ci")
+
+
+def hash_file(x):
+    h = hashlib.sha1()
+    h.update(open(x).read())
+    return h.hexdigest()

src/snapshots.txt

@@ -0,0 +1,4 @@
+S 2011-04-29 7b95b5c
+  linux-i386 f0e166816ce34adc9f7202bd3cfbd80623505f28
+  macos-i386 abf2ee279da63676ca17c9dc9e54d04d8f752b00
+  winnt-i386 7d27adcc5e0c111e3221751962a7df0bcb9a9288