02538d463a
This commit removes all infrastructure from the repository for our so-called snapshots to instead bootstrap the compiler from stable releases. Bootstrapping from a previously stable release is a long-desired feature of distros because they're not fans of downloading binary stage0 blobs from us. Additionally, this makes our own CI easier as we can decommission all of the snapshot builders and start having a regular cadence to when we update the stage0 compiler. A new `src/etc/get-stage0.py` script was added which shares some code with `src/bootstrap/bootstrap.py` to read a new file, `src/stage0.txt`, which lists the current stage0 compiler as well as cargo that we bootstrap from. This script will download the relevant `rustc` package an unpack it into `$target/stage0` as we do today. One problem of bootstrapping from stable releases is that we're not able to compile unstable code (e.g. all the `#![feature]` directives in libcore/libstd). To overcome this we employ two strategies: * The bootstrap key of the previous compiler is hardcoded into `src/stage0.txt` (enabled as a result of #32731) and exported by the build system. This enables nightly features in the compiler we download. * The standard library and compiler are pinned to a specific stage0, which doesn't change, so we're guaranteed that we'll continue compiling as we start from a known fixed source. The process for making a release will also need to be tweaked now to continue to cadence of bootstrapping from the previous release. This process looks like: 1. Merge `beta` to `stable` 2. Produce a new stable compiler. 3. Change `master` to bootstrap from this new stable compiler. 4. Merge `master` to `beta` 5. Produce a new beta compiler 6. Change `master` to bootstrap from this new beta compiler. Step 3 above should involve very few changes as `master` was previously bootstrapping from `beta` which is the same as `stable` at that point in time. Step 6, however, is where we benefit from removing lots of `#[cfg(stage0)]` and get to use new features. This also shouldn't slow the release too much as steps 1-5 requires little work other than waiting and step 6 just needs to happen at some point during a release cycle, it's not time sensitive. Closes #29555 Closes #29557
363 lines
13 KiB
Python
363 lines
13 KiB
Python
# Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
|
|
# file at the top-level directory of this distribution and at
|
|
# http://rust-lang.org/COPYRIGHT.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
# option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
|
|
def get(url, path, verbose=False):
|
|
print("downloading " + url)
|
|
sha_url = url + ".sha256"
|
|
sha_path = path + ".sha256"
|
|
for _url, _path in ((url, path), (sha_url, sha_path)):
|
|
# see http://serverfault.com/questions/301128/how-to-download
|
|
if sys.platform == 'win32':
|
|
run(["PowerShell.exe", "/nologo", "-Command",
|
|
"(New-Object System.Net.WebClient)"
|
|
".DownloadFile('{}', '{}')".format(_url, _path)],
|
|
verbose=verbose)
|
|
else:
|
|
run(["curl", "-o", _path, _url], verbose=verbose)
|
|
print("verifying " + path)
|
|
with open(path, "rb") as f:
|
|
found = hashlib.sha256(f.read()).hexdigest()
|
|
with open(sha_path, "r") as f:
|
|
expected, _ = f.readline().split()
|
|
if found != expected:
|
|
err = ("invalid checksum:\n"
|
|
" found: {}\n"
|
|
" expected: {}".format(found, expected))
|
|
if verbose:
|
|
raise RuntimeError(err)
|
|
sys.exit(err)
|
|
|
|
def unpack(tarball, dst, verbose=False, match=None):
|
|
print("extracting " + tarball)
|
|
fname = os.path.basename(tarball).replace(".tar.gz", "")
|
|
with contextlib.closing(tarfile.open(tarball)) as tar:
|
|
for p in tar.getnames():
|
|
if "/" not in p:
|
|
continue
|
|
name = p.replace(fname + "/", "", 1)
|
|
if match is not None and not name.startswith(match):
|
|
continue
|
|
name = name[len(match) + 1:]
|
|
|
|
fp = os.path.join(dst, name)
|
|
if verbose:
|
|
print(" extracting " + p)
|
|
tar.extract(p, dst)
|
|
tp = os.path.join(dst, p)
|
|
if os.path.isdir(tp) and os.path.exists(fp):
|
|
continue
|
|
shutil.move(tp, fp)
|
|
shutil.rmtree(os.path.join(dst, fname))
|
|
|
|
def run(args, verbose=False):
|
|
if verbose:
|
|
print("running: " + ' '.join(args))
|
|
sys.stdout.flush()
|
|
# Use Popen here instead of call() as it apparently allows powershell on
|
|
# Windows to not lock up waiting for input presumably.
|
|
ret = subprocess.Popen(args)
|
|
code = ret.wait()
|
|
if code != 0:
|
|
err = "failed to run: " + ' '.join(args)
|
|
if verbose:
|
|
raise RuntimeError(err)
|
|
sys.exit(err)
|
|
|
|
def stage0_data(rust_root):
|
|
nightlies = os.path.join(rust_root, "src/stage0.txt")
|
|
with open(nightlies, 'r') as nightlies:
|
|
data = {}
|
|
for line in nightlies.read().split("\n"):
|
|
if line.startswith("#") or line == '':
|
|
continue
|
|
a, b = line.split(": ", 1)
|
|
data[a] = b
|
|
return data
|
|
|
|
class RustBuild:
|
|
def download_stage0(self):
|
|
cache_dst = os.path.join(self.build_dir, "cache")
|
|
rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
|
|
cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
|
|
if not os.path.exists(rustc_cache):
|
|
os.makedirs(rustc_cache)
|
|
if not os.path.exists(cargo_cache):
|
|
os.makedirs(cargo_cache)
|
|
|
|
if self.rustc().startswith(self.bin_root()) and \
|
|
(not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
|
|
if os.path.exists(self.bin_root()):
|
|
shutil.rmtree(self.bin_root())
|
|
channel = self.stage0_rustc_channel()
|
|
filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
|
|
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
|
|
tarball = os.path.join(rustc_cache, filename)
|
|
if not os.path.exists(tarball):
|
|
get(url + "/" + filename, tarball, verbose=self.verbose)
|
|
unpack(tarball, self.bin_root(),
|
|
match="rust-std-" + self.build,
|
|
verbose=self.verbose)
|
|
|
|
filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
|
|
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
|
|
tarball = os.path.join(rustc_cache, filename)
|
|
if not os.path.exists(tarball):
|
|
get(url + "/" + filename, tarball, verbose=self.verbose)
|
|
unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
|
|
with open(self.rustc_stamp(), 'w') as f:
|
|
f.write(self.stage0_rustc_date())
|
|
|
|
if self.cargo().startswith(self.bin_root()) and \
|
|
(not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
|
|
channel = self.stage0_cargo_channel()
|
|
filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
|
|
url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
|
|
tarball = os.path.join(cargo_cache, filename)
|
|
if not os.path.exists(tarball):
|
|
get(url + "/" + filename, tarball, verbose=self.verbose)
|
|
unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
|
|
with open(self.cargo_stamp(), 'w') as f:
|
|
f.write(self.stage0_cargo_date())
|
|
|
|
def stage0_cargo_date(self):
|
|
return self._cargo_date
|
|
|
|
def stage0_cargo_channel(self):
|
|
return self._cargo_channel
|
|
|
|
def stage0_rustc_date(self):
|
|
return self._rustc_date
|
|
|
|
def stage0_rustc_channel(self):
|
|
return self._rustc_channel
|
|
|
|
def rustc_stamp(self):
|
|
return os.path.join(self.bin_root(), '.rustc-stamp')
|
|
|
|
def cargo_stamp(self):
|
|
return os.path.join(self.bin_root(), '.cargo-stamp')
|
|
|
|
def rustc_out_of_date(self):
|
|
if not os.path.exists(self.rustc_stamp()):
|
|
return True
|
|
with open(self.rustc_stamp(), 'r') as f:
|
|
return self.stage0_rustc_date() != f.read()
|
|
|
|
def cargo_out_of_date(self):
|
|
if not os.path.exists(self.cargo_stamp()):
|
|
return True
|
|
with open(self.cargo_stamp(), 'r') as f:
|
|
return self.stage0_cargo_date() != f.read()
|
|
|
|
def bin_root(self):
|
|
return os.path.join(self.build_dir, self.build, "stage0")
|
|
|
|
def get_toml(self, key):
|
|
for line in self.config_toml.splitlines():
|
|
if line.startswith(key + ' ='):
|
|
return self.get_string(line)
|
|
return None
|
|
|
|
def get_mk(self, key):
|
|
for line in iter(self.config_mk.splitlines()):
|
|
if line.startswith(key):
|
|
return line[line.find(':=') + 2:].strip()
|
|
return None
|
|
|
|
def cargo(self):
|
|
config = self.get_toml('cargo')
|
|
if config:
|
|
return config
|
|
return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
|
|
|
|
def rustc(self):
|
|
config = self.get_toml('rustc')
|
|
if config:
|
|
return config
|
|
config = self.get_mk('CFG_LOCAL_RUST')
|
|
if config:
|
|
return config + '/bin/rustc' + self.exe_suffix()
|
|
return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
|
|
|
|
def get_string(self, line):
|
|
start = line.find('"')
|
|
end = start + 1 + line[start+1:].find('"')
|
|
return line[start+1:end]
|
|
|
|
def exe_suffix(self):
|
|
if sys.platform == 'win32':
|
|
return '.exe'
|
|
else:
|
|
return ''
|
|
|
|
def build_bootstrap(self):
|
|
env = os.environ.copy()
|
|
env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
|
|
env["RUSTC"] = self.rustc()
|
|
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
|
|
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
|
|
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
|
|
os.pathsep + env["PATH"]
|
|
self.run([self.cargo(), "build", "--manifest-path",
|
|
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
|
|
env)
|
|
|
|
def run(self, args, env):
|
|
proc = subprocess.Popen(args, env = env)
|
|
ret = proc.wait()
|
|
if ret != 0:
|
|
sys.exit(ret)
|
|
|
|
def build_triple(self):
|
|
config = self.get_toml('build')
|
|
if config:
|
|
return config
|
|
config = self.get_mk('CFG_BUILD')
|
|
if config:
|
|
return config
|
|
try:
|
|
ostype = subprocess.check_output(['uname', '-s']).strip()
|
|
cputype = subprocess.check_output(['uname', '-m']).strip()
|
|
except FileNotFoundError:
|
|
if sys.platform == 'win32':
|
|
return 'x86_64-pc-windows-msvc'
|
|
else:
|
|
err = "uname not found"
|
|
if self.verbose:
|
|
raise Exception(err)
|
|
sys.exit(err)
|
|
|
|
# Darwin's `uname -s` lies and always returns i386. We have to use
|
|
# sysctl instead.
|
|
if ostype == 'Darwin' and cputype == 'i686':
|
|
sysctl = subprocess.check_output(['sysctl', 'hw.optional.x86_64'])
|
|
if sysctl.contains(': 1'):
|
|
cputype = 'x86_64'
|
|
|
|
# The goal here is to come up with the same triple as LLVM would,
|
|
# at least for the subset of platforms we're willing to target.
|
|
if ostype == 'Linux':
|
|
ostype = 'unknown-linux-gnu'
|
|
elif ostype == 'FreeBSD':
|
|
ostype = 'unknown-freebsd'
|
|
elif ostype == 'DragonFly':
|
|
ostype = 'unknown-dragonfly'
|
|
elif ostype == 'Bitrig':
|
|
ostype = 'unknown-bitrig'
|
|
elif ostype == 'OpenBSD':
|
|
ostype = 'unknown-openbsd'
|
|
elif ostype == 'NetBSD':
|
|
ostype = 'unknown-netbsd'
|
|
elif ostype == 'Darwin':
|
|
ostype = 'apple-darwin'
|
|
elif ostype.startswith('MINGW'):
|
|
# msys' `uname` does not print gcc configuration, but prints msys
|
|
# configuration. so we cannot believe `uname -m`:
|
|
# msys1 is always i686 and msys2 is always x86_64.
|
|
# instead, msys defines $MSYSTEM which is MINGW32 on i686 and
|
|
# MINGW64 on x86_64.
|
|
ostype = 'pc-windows-gnu'
|
|
cputype = 'i686'
|
|
if os.environ.get('MSYSTEM') == 'MINGW64':
|
|
cputype = 'x86_64'
|
|
elif ostype.startswith('MSYS'):
|
|
ostype = 'pc-windows-gnu'
|
|
elif ostype.startswith('CYGWIN_NT'):
|
|
cputype = 'i686'
|
|
if ostype.endswith('WOW64'):
|
|
cputype = 'x86_64'
|
|
ostype = 'pc-windows-gnu'
|
|
else:
|
|
err = "unknown OS type: " + ostype
|
|
if self.verbose:
|
|
raise ValueError(err)
|
|
sys.exit(err)
|
|
|
|
if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
|
|
cputype = 'i686'
|
|
elif cputype in {'xscale', 'arm'}:
|
|
cputype = 'arm'
|
|
elif cputype == 'armv7l':
|
|
cputype = 'arm'
|
|
ostype += 'eabihf'
|
|
elif cputype == 'aarch64':
|
|
cputype = 'aarch64'
|
|
elif cputype in {'powerpc', 'ppc', 'ppc64'}:
|
|
cputype = 'powerpc'
|
|
elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
|
|
cputype = 'x86_64'
|
|
else:
|
|
err = "unknown cpu type: " + cputype
|
|
if self.verbose:
|
|
raise ValueError(err)
|
|
sys.exit(err)
|
|
|
|
return cputype + '-' + ostype
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Build rust')
|
|
parser.add_argument('--config')
|
|
parser.add_argument('-v', '--verbose', action='store_true')
|
|
|
|
args = [a for a in sys.argv if a != '-h']
|
|
args, _ = parser.parse_known_args(args)
|
|
|
|
# Configure initial bootstrap
|
|
rb = RustBuild()
|
|
rb.config_toml = ''
|
|
rb.config_mk = ''
|
|
rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
|
|
rb.build_dir = os.path.join(os.getcwd(), "build")
|
|
rb.verbose = args.verbose
|
|
|
|
try:
|
|
with open(args.config or 'config.toml') as config:
|
|
rb.config_toml = config.read()
|
|
except:
|
|
pass
|
|
try:
|
|
rb.config_mk = open('config.mk').read()
|
|
except:
|
|
pass
|
|
|
|
data = stage0_data(rb.rust_root)
|
|
rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
|
|
rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
|
|
|
|
# Fetch/build the bootstrap
|
|
rb.build = rb.build_triple()
|
|
rb.download_stage0()
|
|
sys.stdout.flush()
|
|
rb.build_bootstrap()
|
|
sys.stdout.flush()
|
|
|
|
# Run the bootstrap
|
|
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
|
|
args.append('--src')
|
|
args.append(rb.rust_root)
|
|
args.append('--build')
|
|
args.append(rb.build)
|
|
args.extend(sys.argv[1:])
|
|
env = os.environ.copy()
|
|
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
|
|
rb.run(args, env)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|