mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bootstrap script: slight cleanup

Browse Source
This commit is contained in:
KaDiWa 2023-01-27 22:10:10 +01:00
parent fba9f33c7c
commit 385dbff9e5
No known key found for this signature in database
GPG Key ID: 0B52AE391C674CE5
3 changed files with 152 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
src/bootstrap/bootstrap.py
View File

@ -2,7 +2,6 @@ from __future__ import absolute_import, division, print_function
import argparse
import contextlib
import datetime
import distutils.version
import hashlib
import json
import os
@ -13,17 +12,17 @@ import sys
import tarfile
import tempfile
from time import time, sleep
from time import time
def support_xz():
try:
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
with tarfile.open(temp_path, "w:xz"):
pass
return True
except tarfile.CompressionError:
return False
try:
import lzma
except ImportError:
lzma = None
if sys.platform == 'win32':
EXE_SUFFIX = ".exe"
else:
EXE_SUFFIX = ""
def get(base, url, path, checksums, verbose=False):
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@ -61,7 +60,7 @@ def get(base, url, path, checksums, verbose=False):
def download(path, url, probably_big, verbose):
for _ in range(0, 4):
for _ in range(4):
try:
_download(path, url, probably_big, verbose, True)
return
@ -395,15 +394,15 @@ class RustBuild(object):
def __init__(self):
self.checksums_sha256 = {}
self.stage0_compiler = None
self._download_url = ''
self.download_url = ''
self.build = ''
self.build_dir = ''
self.clean = False
self.config_toml = ''
self.rust_root = ''
self.use_locked_deps = ''
self.use_vendored_sources = ''
self.verbose = False
self.use_locked_deps = False
self.use_vendored_sources = False
self.verbose = 0
self.git_version = None
self.nix_deps_dir = None
@ -426,7 +425,7 @@ class RustBuild(object):
self.program_out_of_date(self.rustc_stamp(), key)):
if os.path.exists(bin_root):
shutil.rmtree(bin_root)
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz'
filename = "rust-std-{}-{}{}".format(
rustc_channel, self.build, tarball_suffix)
pattern = "rust-std-{}".format(self.build)
@ -437,15 +436,17 @@ class RustBuild(object):
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "cargo", tarball_suffix)
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
if self.should_fix_bins_and_dylibs():
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root))
lib_dir = "{}/lib".format(bin_root)
for lib in os.listdir(lib_dir):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root))
lib_dir = "{}/lib".format(bin_root)
for lib in os.listdir(lib_dir):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
with output(self.rustc_stamp()) as rust_stamp:
rust_stamp.write(key)
@ -458,18 +459,53 @@ class RustBuild(object):
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
base = self._download_url
url = "dist/{}".format(key)
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get(
base,
"{}/{}".format(url, filename),
self.download_url,
"dist/{}/{}".format(key, filename),
tarball,
self.checksums_sha256,
verbose=self.verbose,
verbose=self.verbose != 0,
)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose != 0)
def should_fix_bins_and_dylibs(self):
"""Whether or not `fix_bin_or_dylib` needs to be run; can only be True
on NixOS.
"""
default_encoding = sys.getdefaultencoding()
try:
ostype = subprocess.check_output(
['uname', '-s']).strip().decode(default_encoding)
except subprocess.CalledProcessError:
return False
except OSError as reason:
if getattr(reason, 'winerror', None) is not None:
return False
raise reason
if ostype != "Linux":
return False
# If the user has asked binaries to be patched for Nix, then
# don't check for NixOS or `/lib`.
if self.get_toml("patch-binaries-for-nix", "build") == "true":
return True
# Use `/etc/os-release` instead of `/etc/NIXOS`.
# The latter one does not exist on NixOS when using tmpfs as root.
try:
with open("/etc/os-release", "r") as f:
if not any(l.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') for l in f):
return False
except FileNotFoundError:
return False
if os.path.exists("/lib"):
return False
print("info: You seem to be using Nix.")
return True
def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
@ -480,38 +516,7 @@ class RustBuild(object):
Please see https://nixos.org/patchelf.html for more information
"""
default_encoding = sys.getdefaultencoding()
try:
ostype = subprocess.check_output(
['uname', '-s']).strip().decode(default_encoding)
except subprocess.CalledProcessError:
return
except OSError as reason:
if getattr(reason, 'winerror', None) is not None:
return
raise reason
if ostype != "Linux":
return
# If the user has asked binaries to be patched for Nix, then
# don't check for NixOS or `/lib`, just continue to the patching.
if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
# Use `/etc/os-release` instead of `/etc/NIXOS`.
# The latter one does not exist on NixOS when using tmpfs as root.
try:
with open("/etc/os-release", "r") as f:
if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
return
except FileNotFoundError:
return
if os.path.exists("/lib"):
return
# At this point we're pretty sure the user is running NixOS or
# using Nix
nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
print(nix_os_msg, fname)
print("attempting to patch", fname)
# Only build `.nix-deps` once.
nix_deps_dir = self.nix_deps_dir
@ -666,8 +671,7 @@ class RustBuild(object):
config = self.get_toml(program)
if config:
return os.path.expanduser(config)
return os.path.join(self.bin_root(), "bin", "{}{}".format(
program, self.exe_suffix()))
return os.path.join(self.bin_root(), "bin", "{}{}".format(program, EXE_SUFFIX))
@staticmethod
def get_string(line):
@ -692,13 +696,6 @@ class RustBuild(object):
return line[start + 1:end]
return None
@staticmethod
def exe_suffix():
"""Return a suffix for executables"""
if sys.platform == 'win32':
return '.exe'
return ''
def bootstrap_binary(self):
"""Return the path of the bootstrap binary
@ -757,7 +754,6 @@ class RustBuild(object):
if target_linker is not None:
env["RUSTFLAGS"] += " -C linker=" + target_linker
env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
if self.get_toml("deny-warnings", "rust") != "false":
env["RUSTFLAGS"] += " -Dwarnings"
@ -768,8 +764,7 @@ class RustBuild(object):
self.cargo()))
args = [self.cargo(), "build", "--manifest-path",
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
for _ in range(0, self.verbose):
args.append("--verbose")
args.extend("--verbose" for _ in range(self.verbose))
if self.use_locked_deps:
args.append("--locked")
if self.use_vendored_sources:
@ -783,7 +778,7 @@ class RustBuild(object):
args.append("--color=never")
# Run this from the source directory so cargo finds .cargo/config
run(args, env=env, verbose=self.verbose, cwd=self.rust_root)
run(args, env=env, verbose=self.verbose != 0, cwd=self.rust_root)
def build_triple(self):
"""Build triple as in LLVM
@ -792,16 +787,7 @@ class RustBuild(object):
so use `self.build` where possible.
"""
config = self.get_toml('build')
if config:
return config
return default_build_triple(self.verbose)
def set_dist_environment(self, url):
"""Set download URL for normal environment"""
if 'RUSTUP_DIST_SERVER' in os.environ:
self._download_url = os.environ['RUSTUP_DIST_SERVER']
else:
self._download_url = url
return config or default_build_triple(self.verbose != 0)
def check_vendored_status(self):
"""Check that vendoring is configured properly"""
@ -891,7 +877,6 @@ def bootstrap(help_triggered):
build.verbose = max(build.verbose, int(config_verbose))
build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
build.check_vendored_status()
@ -903,8 +888,7 @@ def bootstrap(help_triggered):
data = json.load(f)
build.checksums_sha256 = data["checksums_sha256"]
build.stage0_compiler = Stage0Toolchain(data["compiler"])
build.set_dist_environment(data["config"]["dist_server"])
build.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"]
build.build = args.build or build.build_triple()
@ -932,7 +916,7 @@ def main():
# x.py help <cmd> ...
if len(sys.argv) > 1 and sys.argv[1] == 'help':
sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
sys.argv[1] = '-h'
help_triggered = (
'-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)

133
src/bootstrap/download.rs
View File

@ -70,6 +70,53 @@ pub(crate) fn check_run(&self, cmd: &mut Command) -> bool {
check_run(cmd, self.is_verbose())
}
/// Whether or not `fix_bin_or_dylib` needs to be run; can only be true
/// on NixOS
fn should_fix_bins_and_dylibs(&self) -> bool {
static CACHED: OnceCell<bool> = OnceCell::new();
let val = *CACHED.get_or_init(|| {
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
Err(_) => return false,
Ok(output) if !output.status.success() => return false,
Ok(output) => {
let mut os_name = output.stdout;
if os_name.last() == Some(&b'\n') {
os_name.pop();
}
if os_name != b"Linux" {
return false;
}
}
}
// If the user has asked binaries to be patched for Nix, then
// don't check for NixOS or `/lib`.
// NOTE: this intentionally comes after the Linux check:
// - patchelf only works with ELF files, so no need to run it on Mac or Windows
// - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
if self.patch_binaries_for_nix {
return true;
}
// Use `/etc/os-release` instead of `/etc/NIXOS`.
// The latter one does not exist on NixOS when using tmpfs as root.
let is_nixos = match File::open("/etc/os-release") {
Err(e) if e.kind() == ErrorKind::NotFound => false,
Err(e) => panic!("failed to access /etc/os-release: {}", e),
Ok(os_release) => BufReader::new(os_release).lines().any(|l| {
let l = l.expect("reading /etc/os-release");
matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
}),
};
is_nixos && !Path::new("/lib").exists()
});
if val {
println!("info: You seem to be using Nix.");
}
val
}
/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
/// or the RPATH section, to fix the dynamic library search path
///
@ -78,45 +125,7 @@ pub(crate) fn check_run(&self, cmd: &mut Command) -> bool {
///
/// Please see https://nixos.org/patchelf.html for more information
fn fix_bin_or_dylib(&self, fname: &Path) {
// FIXME: cache NixOS detection?
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
Err(_) => return,
Ok(output) if !output.status.success() => return,
Ok(output) => {
let mut s = output.stdout;
if s.last() == Some(&b'\n') {
s.pop();
}
if s != b"Linux" {
return;
}
}
}
// If the user has asked binaries to be patched for Nix, then
// don't check for NixOS or `/lib`, just continue to the patching.
// NOTE: this intentionally comes after the Linux check:
// - patchelf only works with ELF files, so no need to run it on Mac or Windows
// - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
if !self.patch_binaries_for_nix {
// Use `/etc/os-release` instead of `/etc/NIXOS`.
// The latter one does not exist on NixOS when using tmpfs as root.
const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
let os_release = match File::open("/etc/os-release") {
Err(e) if e.kind() == ErrorKind::NotFound => return,
Err(e) => panic!("failed to access /etc/os-release: {}", e),
Ok(f) => f,
};
if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
return;
}
if Path::new("/lib").exists() {
return;
}
}
// At this point we're pretty sure the user is running NixOS or using Nix
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
println!("attempting to patch {}", fname.display());
// Only build `.nix-deps` once.
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
@ -328,8 +337,10 @@ pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple);
self.download_component(DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0");
self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
if self.should_fix_bins_and_dylibs() {
self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
}
self.create(&rustfmt_stamp, &channel);
Some(rustfmt_path)
@ -358,16 +369,21 @@ pub(crate) fn download_ci_rustc(&self, commit: &str) {
let filename = format!("rust-src-{version}.tar.xz");
self.download_ci_component(filename, "rust-src", commit);
self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
self.fix_bin_or_dylib(&bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"));
let lib_dir = bin_root.join("lib");
for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
let lib = t!(lib);
if lib.path().extension() == Some(OsStr::new("so")) {
self.fix_bin_or_dylib(&lib.path());
if self.should_fix_bins_and_dylibs() {
self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
self.fix_bin_or_dylib(
&bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
);
let lib_dir = bin_root.join("lib");
for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
let lib = t!(lib);
if lib.path().extension() == Some(OsStr::new("so")) {
self.fix_bin_or_dylib(&lib.path());
}
}
}
t!(fs::write(rustc_stamp, commit));
}
}
@ -459,8 +475,10 @@ pub(crate) fn maybe_download_ci_llvm(&self) {
let key = format!("{}{}", llvm_sha, self.llvm_assertions);
if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() {
self.download_ci_llvm(&llvm_sha);
for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
self.fix_bin_or_dylib(&t!(entry).path());
if self.should_fix_bins_and_dylibs() {
for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
self.fix_bin_or_dylib(&t!(entry).path());
}
}
// Update the timestamp of llvm-config to force rustc_llvm to be
@ -475,13 +493,16 @@ pub(crate) fn maybe_download_ci_llvm(&self) {
let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build));
t!(filetime::set_file_times(&llvm_config, now, now));
let llvm_lib = llvm_root.join("lib");
for entry in t!(fs::read_dir(&llvm_lib)) {
let lib = t!(entry).path();
if lib.extension().map_or(false, |ext| ext == "so") {
self.fix_bin_or_dylib(&lib);
if self.should_fix_bins_and_dylibs() {
let llvm_lib = llvm_root.join("lib");
for entry in t!(fs::read_dir(&llvm_lib)) {
let lib = t!(entry).path();
if lib.extension().map_or(false, |ext| ext == "so") {
self.fix_bin_or_dylib(&lib);
}
}
}
t!(fs::write(llvm_stamp, key));
}
}

2
x.py
View File

@ -22,7 +22,7 @@ if sys.version_info.major < 3:
pass
rust_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(rust_dir, "src", "bootstrap"))
sys.path[:0] = [os.path.join(rust_dir, "src", "bootstrap")]
import bootstrap
bootstrap.main()