Rollup merge of #107470 - kadiwa4:bootstrap_cleanup, r=albertlarsan68

Small bootstrap improvements

- i/o-less check for `xz` availability
- cache result of NixOS detection
- load correct `bootstrap` module even when a package of that name is installed
- no `-W semicolon_in_expressions_from_macros` – it is warn-by-default
- one type per variable (making dynamic typing less confusing)
- integrate python-side `--help` flag into the argument parser (makes `-hv` work as a short form of `--help --verbose`)

I even checked that it works with Python 2.

src/bootstrap/bootstrap.py

@@ -2,7 +2,6 @@ from __future__ import absolute_import, division, print_function
 import argparse
 import contextlib
 import datetime
-import distutils.version
 import hashlib
 import json
 import os
@@ -13,17 +12,17 @@ import sys
 import tarfile
 import tempfile
 
-from time import time, sleep
+from time import time
 
-def support_xz():
+try:
-    try:
+    import lzma
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+except ImportError:
-            temp_path = temp_file.name
+    lzma = None
-        with tarfile.open(temp_path, "w:xz"):
+
-            pass
+if sys.platform == 'win32':
-        return True
+    EXE_SUFFIX = ".exe"
-    except tarfile.CompressionError:
+else:
-        return False
+    EXE_SUFFIX = ""
 
 def get(base, url, path, checksums, verbose=False):
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -61,7 +60,7 @@ def get(base, url, path, checksums, verbose=False):
 
 
 def download(path, url, probably_big, verbose):
-    for _ in range(0, 4):
+    for _ in range(4):
         try:
             _download(path, url, probably_big, verbose, True)
             return
@@ -395,17 +394,18 @@ class RustBuild(object):
     def __init__(self):
         self.checksums_sha256 = {}
         self.stage0_compiler = None
-        self._download_url = ''
+        self.download_url = ''
         self.build = ''
         self.build_dir = ''
         self.clean = False
         self.config_toml = ''
         self.rust_root = ''
-        self.use_locked_deps = ''
+        self.use_locked_deps = False
-        self.use_vendored_sources = ''
+        self.use_vendored_sources = False
         self.verbose = False
         self.git_version = None
         self.nix_deps_dir = None
+        self._should_fix_bins_and_dylibs = None
 
     def download_toolchain(self):
         """Fetch the build system for Rust, written in Rust
@@ -426,7 +426,7 @@ class RustBuild(object):
                  self.program_out_of_date(self.rustc_stamp(), key)):
             if os.path.exists(bin_root):
                 shutil.rmtree(bin_root)
-            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
+            tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz'
             filename = "rust-std-{}-{}{}".format(
                 rustc_channel, self.build, tarball_suffix)
             pattern = "rust-std-{}".format(self.build)
@@ -437,15 +437,17 @@ class RustBuild(object):
             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
                                             tarball_suffix)
             self._download_component_helper(filename, "cargo", tarball_suffix)
-            self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
+            if self.should_fix_bins_and_dylibs():
+                self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
+
+                self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
+                self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
+                self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root))
+                lib_dir = "{}/lib".format(bin_root)
+                for lib in os.listdir(lib_dir):
+                    if lib.endswith(".so"):
+                        self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 
-            self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
-            self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
-            self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root))
-            lib_dir = "{}/lib".format(bin_root)
-            for lib in os.listdir(lib_dir):
-                if lib.endswith(".so"):
-                    self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
             with output(self.rustc_stamp()) as rust_stamp:
                 rust_stamp.write(key)
 
@@ -458,19 +460,62 @@ class RustBuild(object):
         if not os.path.exists(rustc_cache):
             os.makedirs(rustc_cache)
 
-        base = self._download_url
-        url = "dist/{}".format(key)
         tarball = os.path.join(rustc_cache, filename)
         if not os.path.exists(tarball):
             get(
-                base,
+                self.download_url,
-                "{}/{}".format(url, filename),
+                "dist/{}/{}".format(key, filename),
                 tarball,
                 self.checksums_sha256,
                 verbose=self.verbose,
             )
         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 
+    def should_fix_bins_and_dylibs(self):
+        """Whether or not `fix_bin_or_dylib` needs to be run; can only be True
+        on NixOS.
+        """
+        if self._should_fix_bins_and_dylibs is not None:
+            return self._should_fix_bins_and_dylibs
+
+        def get_answer():
+            default_encoding = sys.getdefaultencoding()
+            try:
+                ostype = subprocess.check_output(
+                    ['uname', '-s']).strip().decode(default_encoding)
+            except subprocess.CalledProcessError:
+                return False
+            except OSError as reason:
+                if getattr(reason, 'winerror', None) is not None:
+                    return False
+                raise reason
+
+            if ostype != "Linux":
+                return False
+
+            # If the user has asked binaries to be patched for Nix, then
+            # don't check for NixOS or `/lib`.
+            if self.get_toml("patch-binaries-for-nix", "build") == "true":
+                return True
+
+            # Use `/etc/os-release` instead of `/etc/NIXOS`.
+            # The latter one does not exist on NixOS when using tmpfs as root.
+            try:
+                with open("/etc/os-release", "r") as f:
+                    if not any(l.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') for l in f):
+                        return False
+            except FileNotFoundError:
+                return False
+            if os.path.exists("/lib"):
+                return False
+
+            return True
+
+        answer = self._should_fix_bins_and_dylibs = get_answer()
+        if answer:
+            print("info: You seem to be using Nix.")
+        return answer
+
     def fix_bin_or_dylib(self, fname):
         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
         or the RPATH section, to fix the dynamic library search path
@@ -480,38 +525,8 @@ class RustBuild(object):
 
         Please see https://nixos.org/patchelf.html for more information
         """
-        default_encoding = sys.getdefaultencoding()
+        assert self._should_fix_bins_and_dylibs is True
-        try:
+        print("attempting to patch", fname)
-            ostype = subprocess.check_output(
-                ['uname', '-s']).strip().decode(default_encoding)
-        except subprocess.CalledProcessError:
-            return
-        except OSError as reason:
-            if getattr(reason, 'winerror', None) is not None:
-                return
-            raise reason
-
-        if ostype != "Linux":
-            return
-
-        # If the user has asked binaries to be patched for Nix, then
-        # don't check for NixOS or `/lib`, just continue to the patching.
-        if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
-            # Use `/etc/os-release` instead of `/etc/NIXOS`.
-            # The latter one does not exist on NixOS when using tmpfs as root.
-            try:
-                with open("/etc/os-release", "r") as f:
-                    if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
-                        return
-            except FileNotFoundError:
-                return
-            if os.path.exists("/lib"):
-                return
-
-        # At this point we're pretty sure the user is running NixOS or
-        # using Nix
-        nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
-        print(nix_os_msg, fname)
 
         # Only build `.nix-deps` once.
         nix_deps_dir = self.nix_deps_dir
@@ -666,8 +681,7 @@ class RustBuild(object):
         config = self.get_toml(program)
         if config:
             return os.path.expanduser(config)
-        return os.path.join(self.bin_root(), "bin", "{}{}".format(
+        return os.path.join(self.bin_root(), "bin", "{}{}".format(program, EXE_SUFFIX))
-            program, self.exe_suffix()))
 
     @staticmethod
     def get_string(line):
@@ -692,13 +706,6 @@ class RustBuild(object):
             return line[start + 1:end]
         return None
 
-    @staticmethod
-    def exe_suffix():
-        """Return a suffix for executables"""
-        if sys.platform == 'win32':
-            return '.exe'
-        return ''
-
     def bootstrap_binary(self):
         """Return the path of the bootstrap binary
 
@@ -710,7 +717,7 @@ class RustBuild(object):
         """
         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 
-    def build_bootstrap(self, color):
+    def build_bootstrap(self, color, verbose_count):
         """Build bootstrap"""
         print("Building bootstrap")
         build_dir = os.path.join(self.build_dir, "bootstrap")
@@ -757,7 +764,6 @@ class RustBuild(object):
         if target_linker is not None:
             env["RUSTFLAGS"] += " -C linker=" + target_linker
         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
-        env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
         if self.get_toml("deny-warnings", "rust") != "false":
             env["RUSTFLAGS"] += " -Dwarnings"
 
@@ -768,8 +774,7 @@ class RustBuild(object):
                 self.cargo()))
         args = [self.cargo(), "build", "--manifest-path",
                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
-        for _ in range(0, self.verbose):
+        args.extend("--verbose" for _ in range(verbose_count))
-            args.append("--verbose")
         if self.use_locked_deps:
             args.append("--locked")
         if self.use_vendored_sources:
@@ -792,16 +797,7 @@ class RustBuild(object):
         so use `self.build` where possible.
         """
         config = self.get_toml('build')
-        if config:
+        return config or default_build_triple(self.verbose)
-            return config
-        return default_build_triple(self.verbose)
-
-    def set_dist_environment(self, url):
-        """Set download URL for normal environment"""
-        if 'RUSTUP_DIST_SERVER' in os.environ:
-            self._download_url = os.environ['RUSTUP_DIST_SERVER']
-        else:
-            self._download_url = url
 
     def check_vendored_status(self):
         """Check that vendoring is configured properly"""
@@ -834,17 +830,10 @@ class RustBuild(object):
             if os.path.exists(cargo_dir):
                 shutil.rmtree(cargo_dir)
 
-def bootstrap(help_triggered):
+def parse_args():
-    """Configure, fetch, build and run the initial bootstrap"""
+    """Parse the command line arguments that the python script needs."""
-
+    parser = argparse.ArgumentParser(add_help=False)
-    # If the user is asking for help, let them know that the whole download-and-build
+    parser.add_argument('-h', '--help', action='store_true')
-    # process has to happen before anything is printed out.
-    if help_triggered:
-        print("info: Downloading and building bootstrap before processing --help")
-        print("      command. See src/bootstrap/README.md for help with common")
-        print("      commands.")
-
-    parser = argparse.ArgumentParser(description='Build rust')
     parser.add_argument('--config')
     parser.add_argument('--build-dir')
     parser.add_argument('--build')
@@ -852,13 +841,14 @@ def bootstrap(help_triggered):
     parser.add_argument('--clean', action='store_true')
     parser.add_argument('-v', '--verbose', action='count', default=0)
 
-    args = [a for a in sys.argv if a != '-h' and a != '--help']
+    return parser.parse_known_args(sys.argv)[0]
-    args, _ = parser.parse_known_args(args)
 
+def bootstrap(args):
+    """Configure, fetch, build and run the initial bootstrap"""
     # Configure initial bootstrap
     build = RustBuild()
     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
-    build.verbose = args.verbose
+    build.verbose = args.verbose != 0
     build.clean = args.clean
 
     # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
@@ -886,12 +876,12 @@ def bootstrap(help_triggered):
         with open(include_path) as included_toml:
             build.config_toml += os.linesep + included_toml.read()
 
-    config_verbose = build.get_toml('verbose', 'build')
+    verbose_count = args.verbose
-    if config_verbose is not None:
+    config_verbose_count = build.get_toml('verbose', 'build')
-        build.verbose = max(build.verbose, int(config_verbose))
+    if config_verbose_count is not None:
+        verbose_count = max(args.verbose, int(config_verbose_count))
 
     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
-
     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 
     build.check_vendored_status()
@@ -903,8 +893,7 @@ def bootstrap(help_triggered):
         data = json.load(f)
     build.checksums_sha256 = data["checksums_sha256"]
     build.stage0_compiler = Stage0Toolchain(data["compiler"])
-
+    build.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"]
-    build.set_dist_environment(data["config"]["dist_server"])
 
     build.build = args.build or build.build_triple()
 
@@ -914,7 +903,7 @@ def bootstrap(help_triggered):
     # Fetch/build the bootstrap
     build.download_toolchain()
     sys.stdout.flush()
-    build.build_bootstrap(args.color)
+    build.build_bootstrap(args.color, verbose_count)
     sys.stdout.flush()
 
     # Run the bootstrap
@@ -932,25 +921,32 @@ def main():
 
     # x.py help <cmd> ...
     if len(sys.argv) > 1 and sys.argv[1] == 'help':
-        sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
+        sys.argv[1] = '-h'
 
-    help_triggered = (
+    args = parse_args()
-        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
+    help_triggered = args.help or len(sys.argv) == 1
+
+    # If the user is asking for help, let them know that the whole download-and-build
+    # process has to happen before anything is printed out.
+    if help_triggered:
+        print(
+            "info: Downloading and building bootstrap before processing --help command.\n"
+            "      See src/bootstrap/README.md for help with common commands."
+        )
+
+    exit_code = 0
     try:
-        bootstrap(help_triggered)
+        bootstrap(args)
-        if not help_triggered:
-            print("Build completed successfully in {}".format(
-                format_build_time(time() - start_time)))
     except (SystemExit, KeyboardInterrupt) as error:
         if hasattr(error, 'code') and isinstance(error.code, int):
             exit_code = error.code
         else:
             exit_code = 1
             print(error)
-        if not help_triggered:
+
-            print("Build completed unsuccessfully in {}".format(
+    if not help_triggered:
-                format_build_time(time() - start_time)))
+        print("Build completed successfully in", format_build_time(time() - start_time))
-        sys.exit(exit_code)
+    sys.exit(exit_code)
 
 
 if __name__ == '__main__':

src/bootstrap/download.rs

@@ -18,6 +18,8 @@ use crate::{
     Config,
 };
 
+static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell<bool> = OnceCell::new();
+
 /// Generic helpers that are useful anywhere in bootstrap.
 impl Config {
     pub fn is_verbose(&self) -> bool {
@@ -70,6 +72,51 @@ impl Config {
         check_run(cmd, self.is_verbose())
     }
 
+    /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true
+    /// on NixOS
+    fn should_fix_bins_and_dylibs(&self) -> bool {
+        let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| {
+            match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
+                Err(_) => return false,
+                Ok(output) if !output.status.success() => return false,
+                Ok(output) => {
+                    let mut os_name = output.stdout;
+                    if os_name.last() == Some(&b'\n') {
+                        os_name.pop();
+                    }
+                    if os_name != b"Linux" {
+                        return false;
+                    }
+                }
+            }
+
+            // If the user has asked binaries to be patched for Nix, then
+            // don't check for NixOS or `/lib`.
+            // NOTE: this intentionally comes after the Linux check:
+            // - patchelf only works with ELF files, so no need to run it on Mac or Windows
+            // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
+            if self.patch_binaries_for_nix {
+                return true;
+            }
+
+            // Use `/etc/os-release` instead of `/etc/NIXOS`.
+            // The latter one does not exist on NixOS when using tmpfs as root.
+            let is_nixos = match File::open("/etc/os-release") {
+                Err(e) if e.kind() == ErrorKind::NotFound => false,
+                Err(e) => panic!("failed to access /etc/os-release: {}", e),
+                Ok(os_release) => BufReader::new(os_release).lines().any(|l| {
+                    let l = l.expect("reading /etc/os-release");
+                    matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
+                }),
+            };
+            is_nixos && !Path::new("/lib").exists()
+        });
+        if val {
+            println!("info: You seem to be using Nix.");
+        }
+        val
+    }
+
     /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
     /// or the RPATH section, to fix the dynamic library search path
     ///
@@ -78,45 +125,8 @@ impl Config {
     ///
     /// Please see https://nixos.org/patchelf.html for more information
     fn fix_bin_or_dylib(&self, fname: &Path) {
-        // FIXME: cache NixOS detection?
+        assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true));
-        match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
+        println!("attempting to patch {}", fname.display());
-            Err(_) => return,
-            Ok(output) if !output.status.success() => return,
-            Ok(output) => {
-                let mut s = output.stdout;
-                if s.last() == Some(&b'\n') {
-                    s.pop();
-                }
-                if s != b"Linux" {
-                    return;
-                }
-            }
-        }
-
-        // If the user has asked binaries to be patched for Nix, then
-        // don't check for NixOS or `/lib`, just continue to the patching.
-        // NOTE: this intentionally comes after the Linux check:
-        // - patchelf only works with ELF files, so no need to run it on Mac or Windows
-        // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
-        if !self.patch_binaries_for_nix {
-            // Use `/etc/os-release` instead of `/etc/NIXOS`.
-            // The latter one does not exist on NixOS when using tmpfs as root.
-            const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
-            let os_release = match File::open("/etc/os-release") {
-                Err(e) if e.kind() == ErrorKind::NotFound => return,
-                Err(e) => panic!("failed to access /etc/os-release: {}", e),
-                Ok(f) => f,
-            };
-            if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
-                return;
-            }
-            if Path::new("/lib").exists() {
-                return;
-            }
-        }
-
-        // At this point we're pretty sure the user is running NixOS or using Nix
-        println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
 
         // Only build `.nix-deps` once.
         static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
@@ -340,8 +350,10 @@ impl Config {
             "rustfmt",
         );
 
-        self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
+        if self.should_fix_bins_and_dylibs() {
-        self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
+            self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
+            self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
+        }
 
         self.create(&rustfmt_stamp, &channel);
         Some(rustfmt_path)
@@ -370,16 +382,21 @@ impl Config {
             let filename = format!("rust-src-{version}.tar.xz");
             self.download_ci_component(filename, "rust-src", commit);
 
-            self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
+            if self.should_fix_bins_and_dylibs() {
-            self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
+                self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
-            self.fix_bin_or_dylib(&bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"));
+                self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
-            let lib_dir = bin_root.join("lib");
+                self.fix_bin_or_dylib(
-            for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
+                    &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
-                let lib = t!(lib);
+                );
-                if lib.path().extension() == Some(OsStr::new("so")) {
+                let lib_dir = bin_root.join("lib");
-                    self.fix_bin_or_dylib(&lib.path());
+                for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
+                    let lib = t!(lib);
+                    if lib.path().extension() == Some(OsStr::new("so")) {
+                        self.fix_bin_or_dylib(&lib.path());
+                    }
                 }
             }
+
             t!(fs::write(rustc_stamp, commit));
         }
     }
@@ -471,8 +488,10 @@ impl Config {
         let key = format!("{}{}", llvm_sha, self.llvm_assertions);
         if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() {
             self.download_ci_llvm(&llvm_sha);
-            for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
+            if self.should_fix_bins_and_dylibs() {
-                self.fix_bin_or_dylib(&t!(entry).path());
+                for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
+                    self.fix_bin_or_dylib(&t!(entry).path());
+                }
             }
 
             // Update the timestamp of llvm-config to force rustc_llvm to be
@@ -487,13 +506,16 @@ impl Config {
             let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build));
             t!(filetime::set_file_times(&llvm_config, now, now));
 
-            let llvm_lib = llvm_root.join("lib");
+            if self.should_fix_bins_and_dylibs() {
-            for entry in t!(fs::read_dir(&llvm_lib)) {
+                let llvm_lib = llvm_root.join("lib");
-                let lib = t!(entry).path();
+                for entry in t!(fs::read_dir(&llvm_lib)) {
-                if lib.extension().map_or(false, |ext| ext == "so") {
+                    let lib = t!(entry).path();
-                    self.fix_bin_or_dylib(&lib);
+                    if lib.extension().map_or(false, |ext| ext == "so") {
+                        self.fix_bin_or_dylib(&lib);
+                    }
                 }
             }
+
             t!(fs::write(llvm_stamp, key));
         }
     }

x.py

@@ -22,7 +22,8 @@ if sys.version_info.major < 3:
             pass
 
 rust_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(rust_dir, "src", "bootstrap"))
+# For the import below, have Python search in src/bootstrap first.
+sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap"))
 
 import bootstrap
 bootstrap.main()