diff --git a/src/etc/get-snapshot.py b/src/etc/get-snapshot.py new file mode 100755 index 00000000000..d9547b2f8df --- /dev/null +++ b/src/etc/get-snapshot.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +import os, tarfile, hashlib, re, shutil +from snapshot import * + +def snap_filename_hash_part(snap): + match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap) + if not match: + raise Exception("unable to find hash in filename: " + snap) + return match.group(1) + +def get_snapshot_and_check_hash(snap): + + hsh = snap_filename_hash_part(snap) + + h = hashlib.sha1() + url = download_url_base + "/" + snap + print "downloading " + url + u = urllib2.urlopen(url) + print "checking hash on download" + data = u.read() + h.update(data) + if h.hexdigest() != hsh: + raise Exception("hash check failed on " + snap) + + print "hash ok" + with open(os.path.join(download_dir_base, snap), "w+b") as f: + f.write(data) + return True + +def unpack_snapshot(snap): + dl_path = os.path.join(download_dir_base, snap) + print "opening snapshot " + dl_path + tar = tarfile.open(dl_path) + kernel = get_kernel() + for name in snapshot_files[kernel]: + p = os.path.join("rust-stage0", name) + fp = os.path.join("stage0", name) + print "extracting " + fp + tar.extract(p, download_unpack_base) + tp = os.path.join(download_unpack_base, p) + shutil.move(tp, fp) + tar.close() + shutil.rmtree(download_unpack_base) + +def determine_last_snapshot_for_platform(): + lines = open(snapshotfile).readlines(); + + platform = get_platform() + + found = False + hsh = None + date = None + rev = None + + for ln in range(len(lines) - 1, -1, -1): + parsed = parse_line(ln, lines[ln]) + if (not parsed): continue + + if parsed["type"] == "file": + if parsed["platform"] == platform: + hsh = parsed["hash"] + elif parsed["type"] == "snapshot": + date = parsed["date"] + rev = parsed["rev"] + found = True + break + elif parsed["type"] == "transition" and not foundSnapshot: + raise Exception("working on a transition, not updating stage0") + + if not found: + raise Exception("no snapshot entries in file") + + if not hsh: + raise Exception("no snapshot file found for platform %s, rev %s" % + (platform, rev)) + + return full_snapshot_name(date, rev, get_kernel(), get_cpu(), hsh) + +# Main + +snap = determine_last_snapshot_for_platform() +print "determined most recent snapshot: " + snap +dl = os.path.join(download_dir_base, snap) +if (os.path.exists(dl)): + if (snap_filename_hash_part(snap) == hash_file(dl)): + print "found existing download with ok hash" + else: + print "bad hash on existing download, re-fetching" + get_snapshot_and_check_hash(snap) +else: + print "no cached download, fetching" + get_snapshot_and_check_hash(snap) + +unpack_snapshot(snap) diff --git a/src/etc/make-snapshot.py b/src/etc/make-snapshot.py new file mode 100755 index 00000000000..3d6c48e3c53 --- /dev/null +++ b/src/etc/make-snapshot.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +import shutil, tarfile +from snapshot import * + +kernel = get_kernel() +cpu = get_cpu() +rev = local_rev_short_sha() +date = local_rev_committer_date().split()[0] + +file0 = partial_snapshot_name(date, rev, kernel, cpu) + +tar = tarfile.open(file0, "w:bz2") +for name in snapshot_files[kernel]: + tar.add(os.path.join("stage2", name), + os.path.join("rust-stage0", name)) +tar.close() + +h = hash_file(file0) +file1 = full_snapshot_name(date, rev, kernel, cpu, h) + +shutil.move(file0, file1) + +print file1 diff --git a/src/etc/snapshot.py b/src/etc/snapshot.py new file mode 100644 index 00000000000..2aa241c57a6 --- /dev/null +++ b/src/etc/snapshot.py @@ -0,0 +1,89 @@ +import re, os, sys, hashlib, tarfile, shutil, subprocess, urllib2, tempfile + +snapshotfile = "snapshots.txt" +download_url_base = "http://dl.rust-lang.org/stage0-snapshots" +download_dir_base = "dl" +download_unpack_base = os.path.join(download_dir_base, "unpack") + +snapshot_files = { + "linux": ["rustc", "glue.o", "libstd.so" ], + "macos": ["rustc", "glue.o", "libstd.dylib" ], + "winnt": ["rustc.exe", "glue.o", "std.dll" ] + } + +def parse_line(n, line): + global snapshotfile + + if re.match(r"\s*$", line): return None + + match = re.match(r"\s+([\w_-]+) ([a-fA-F\d]{40})\s*$", line) + if match: + return { "type": "file", + "platform": match.group(1), + "hash": match.group(2).lower() } + + match = re.match(r"([ST]) (\d{4}-\d{2}-\d{2}) ([a-fA-F\d]+)\s*$", line); + if (not match): + raise Exception("%s:%d:E syntax error" % (snapshotfile, n)) + ttype = "snapshot" + if (match.group(1) == "T"): + ttype = "transition" + return {"type": ttype, + "date": match.group(2), + "rev": match.group(3)} + + +def partial_snapshot_name(date, rev, kernel, cpu): + return ("rust-stage0-%s-%s-%s-%s.tar.bz2" + % (date, rev, kernel, cpu)) + +def full_snapshot_name(date, rev, kernel, cpu, hsh): + return ("rust-stage0-%s-%s-%s-%s-%s.tar.bz2" + % (date, rev, kernel, cpu, hsh)) + + +def get_kernel(): + if os.name == "nt": + return "winnt" + kernel = os.uname()[0].lower() + if kernel == "darwin": + kernel = "macos" + return kernel + + +def get_cpu(): + # return os.uname()[-1].lower() + return "i386" + + +def get_platform(): + return "%s-%s" % (get_kernel(), get_cpu()) + + +def cmd_out(cmdline): + p = subprocess.Popen(cmdline, + stdout=subprocess.PIPE) + return p.communicate()[0].strip() + + +def local_rev_info(field): + return cmd_out(["git", "log", "-n", "1", + "--format=%%%s" % field, "HEAD"]) + + +def local_rev_full_sha(): + return local_rev_info("H").split()[0] + + +def local_rev_short_sha(): + return local_rev_info("h").split()[0] + + +def local_rev_committer_date(): + return local_rev_info("ci") + + +def hash_file(x): + h = hashlib.sha1() + h.update(open(x).read()) + return h.hexdigest() diff --git a/src/snapshots.txt b/src/snapshots.txt new file mode 100644 index 00000000000..436015b056c --- /dev/null +++ b/src/snapshots.txt @@ -0,0 +1,4 @@ +S 2011-04-29 7b95b5c + linux-i386 f0e166816ce34adc9f7202bd3cfbd80623505f28 + macos-i386 abf2ee279da63676ca17c9dc9e54d04d8f752b00 + winnt-i386 7d27adcc5e0c111e3221751962a7df0bcb9a9288