From 80b81adc63c4797cac217a586fb4054697a2c70e Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Aug 2021 11:26:03 +0200 Subject: [PATCH] switch stage0.txt to stage0.json and add a tool to generate it --- Cargo.lock | 11 ++ Cargo.toml | 1 + src/bootstrap/bootstrap.py | 76 ++++++-------- src/bootstrap/bootstrap_test.py | 20 ---- src/bootstrap/builder.rs | 2 +- src/bootstrap/lib.rs | 2 +- src/bootstrap/run.rs | 21 ++++ src/bootstrap/sanity.rs | 12 +-- src/bootstrap/tool.rs | 1 + src/stage0.json | 12 +++ src/stage0.txt | 42 -------- src/tools/bump-stage0/Cargo.toml | 13 +++ src/tools/bump-stage0/src/main.rs | 167 ++++++++++++++++++++++++++++++ 13 files changed, 263 insertions(+), 117 deletions(-) create mode 100644 src/stage0.json delete mode 100644 src/stage0.txt create mode 100644 src/tools/bump-stage0/Cargo.toml create mode 100644 src/tools/bump-stage0/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 9e0624c57ae..21b6e41dba3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,17 @@ dependencies = [ name = "build_helper" version = "0.1.0" +[[package]] +name = "bump-stage0" +version = "0.1.0" +dependencies = [ + "anyhow", + "curl", + "serde", + "serde_json", + "toml", +] + [[package]] name = "byte-tools" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index dedfe45aca4..3822da2ccd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ members = [ "src/tools/expand-yaml-anchors", "src/tools/jsondocck", "src/tools/html-checker", + "src/tools/bump-stage0", ] exclude = [ diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 3faf38c66ec..3160f6c57d8 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -4,6 +4,7 @@ import contextlib import datetime import distutils.version import hashlib +import json import os import re import shutil @@ -176,15 +177,6 @@ def require(cmd, exit=True): sys.exit(1) -def stage0_data(rust_root): - """Build a dictionary from stage0.txt""" - nightlies = os.path.join(rust_root, "src/stage0.txt") - with open(nightlies, 'r') as nightlies: - lines = [line.rstrip() for line in nightlies - if not line.startswith("#")] - return dict([line.split(": ", 1) for line in lines if line]) - - def format_build_time(duration): """Return a nicer format for build time @@ -371,13 +363,21 @@ def output(filepath): os.rename(tmp, filepath) +class Stage0Toolchain: + def __init__(self, stage0_payload): + self.date = stage0_payload["date"] + self.version = stage0_payload["version"] + + def channel(self): + return self.version + "-" + self.date + + class RustBuild(object): """Provide all the methods required to build Rust""" def __init__(self): - self.date = '' + self.stage0_compiler = None + self.stage0_rustfmt = None self._download_url = '' - self.rustc_channel = '' - self.rustfmt_channel = '' self.build = '' self.build_dir = '' self.clean = False @@ -401,11 +401,10 @@ class RustBuild(object): will move all the content to the right place. """ if rustc_channel is None: - rustc_channel = self.rustc_channel - rustfmt_channel = self.rustfmt_channel + rustc_channel = self.stage0_compiler.version bin_root = self.bin_root(stage0) - key = self.date + key = self.stage0_compiler.date if not stage0: key += str(self.rustc_commit) if self.rustc(stage0).startswith(bin_root) and \ @@ -444,19 +443,23 @@ class RustBuild(object): if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( not os.path.exists(self.rustfmt()) - or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel) + or self.program_out_of_date( + self.rustfmt_stamp(), + "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel() + ) ): - if rustfmt_channel: + if self.stage0_rustfmt is not None: tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' - [channel, date] = rustfmt_channel.split('-', 1) - filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix) + filename = "rustfmt-{}-{}{}".format( + self.stage0_rustfmt.version, self.build, tarball_suffix, + ) self._download_component_helper( - filename, "rustfmt-preview", tarball_suffix, key=date + filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date ) self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root)) self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root)) with output(self.rustfmt_stamp()) as rustfmt_stamp: - rustfmt_stamp.write(self.rustfmt_channel) + rustfmt_stamp.write(self.stage0_rustfmt.channel()) # Avoid downloading LLVM twice (once for stage0 and once for the master rustc) if self.downloading_llvm() and stage0: @@ -517,7 +520,7 @@ class RustBuild(object): ): if key is None: if stage0: - key = self.date + key = self.stage0_compiler.date else: key = self.rustc_commit cache_dst = os.path.join(self.build_dir, "cache") @@ -815,7 +818,7 @@ class RustBuild(object): def rustfmt(self): """Return config path for rustfmt""" - if not self.rustfmt_channel: + if self.stage0_rustfmt is None: return None return self.program_config('rustfmt') @@ -1039,19 +1042,12 @@ class RustBuild(object): self.update_submodule(module[0], module[1], recorded_submodules) print("Submodules updated in %.2f seconds" % (time() - start_time)) - def set_normal_environment(self): + def set_dist_environment(self, url): """Set download URL for normal environment""" if 'RUSTUP_DIST_SERVER' in os.environ: self._download_url = os.environ['RUSTUP_DIST_SERVER'] else: - self._download_url = 'https://static.rust-lang.org' - - def set_dev_environment(self): - """Set download URL for development environment""" - if 'RUSTUP_DEV_DIST_SERVER' in os.environ: - self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER'] - else: - self._download_url = 'https://dev-static.rust-lang.org' + self._download_url = url def check_vendored_status(self): """Check that vendoring is configured properly""" @@ -1160,17 +1156,13 @@ def bootstrap(help_triggered): build_dir = build.get_toml('build-dir', 'build') or 'build' build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root)) - data = stage0_data(build.rust_root) - build.date = data['date'] - build.rustc_channel = data['rustc'] + with open(os.path.join(build.rust_root, "src", "stage0.json")) as f: + data = json.load(f) + build.stage0_compiler = Stage0Toolchain(data["compiler"]) + if data.get("rustfmt") is not None: + build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"]) - if "rustfmt" in data: - build.rustfmt_channel = data['rustfmt'] - - if 'dev' in data: - build.set_dev_environment() - else: - build.set_normal_environment() + build.set_dist_environment(data["dist_server"]) build.build = args.build or build.build_triple() build.update_submodules() diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py index 61507114159..c91fd60da2f 100644 --- a/src/bootstrap/bootstrap_test.py +++ b/src/bootstrap/bootstrap_test.py @@ -13,25 +13,6 @@ from shutil import rmtree import bootstrap -class Stage0DataTestCase(unittest.TestCase): - """Test Case for stage0_data""" - def setUp(self): - self.rust_root = tempfile.mkdtemp() - os.mkdir(os.path.join(self.rust_root, "src")) - with open(os.path.join(self.rust_root, "src", - "stage0.txt"), "w") as stage0: - stage0.write("#ignore\n\ndate: 2017-06-15\nrustc: beta\ncargo: beta\nrustfmt: beta") - - def tearDown(self): - rmtree(self.rust_root) - - def test_stage0_data(self): - """Extract data from stage0.txt""" - expected = {"date": "2017-06-15", "rustc": "beta", "cargo": "beta", "rustfmt": "beta"} - data = bootstrap.stage0_data(self.rust_root) - self.assertDictEqual(data, expected) - - class VerifyTestCase(unittest.TestCase): """Test Case for verify""" def setUp(self): @@ -99,7 +80,6 @@ if __name__ == '__main__': TEST_LOADER = unittest.TestLoader() SUITE.addTest(doctest.DocTestSuite(bootstrap)) SUITE.addTests([ - TEST_LOADER.loadTestsFromTestCase(Stage0DataTestCase), TEST_LOADER.loadTestsFromTestCase(VerifyTestCase), TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)]) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 5911309a044..0a6ed2f49b7 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -523,7 +523,7 @@ macro_rules! describe { install::Src, install::Rustc ), - Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest), + Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest, run::BumpStage0), } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 3d56650f775..a4735d54be0 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -31,7 +31,7 @@ //! When you execute `x.py build`, the steps executed are: //! //! * First, the python script is run. This will automatically download the -//! stage0 rustc and cargo according to `src/stage0.txt`, or use the cached +//! stage0 rustc and cargo according to `src/stage0.json`, or use the cached //! versions if they're available. These are then used to compile rustbuild //! itself (using Cargo). Finally, control is then transferred to rustbuild. //! diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index 7c64e5a0aad..11b393857e7 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -82,3 +82,24 @@ fn run(self, builder: &Builder<'_>) { builder.run(&mut cmd); } } + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BumpStage0; + +impl Step for BumpStage0 { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/bump-stage0") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BumpStage0); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut cmd = builder.tool_cmd(Tool::BumpStage0); + builder.run(&mut cmd); + } +} diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 74e50c60610..d7db2cef24f 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -15,7 +15,7 @@ use std::path::PathBuf; use std::process::Command; -use build_helper::{output, t}; +use build_helper::output; use crate::cache::INTERNER; use crate::config::Target; @@ -227,14 +227,4 @@ pub fn check(build: &mut Build) { if let Some(ref s) = build.config.ccache { cmd_finder.must_have(s); } - - if build.config.channel == "stable" { - let stage0 = t!(fs::read_to_string(build.src.join("src/stage0.txt"))); - if stage0.contains("\ndev:") { - panic!( - "bootstrapping from a dev compiler in a stable release, but \ - should only be bootstrapping from a released compiler!" - ); - } - } } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index f5e3f61dcc8..c0358946385 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -377,6 +377,7 @@ fn run(self, builder: &Builder<'_>) -> PathBuf { LintDocs, "src/tools/lint-docs", "lint-docs"; JsonDocCk, "src/tools/jsondocck", "jsondocck"; HtmlChecker, "src/tools/html-checker", "html-checker"; + BumpStage0, "src/tools/bump-stage0", "bump-stage0"; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] diff --git a/src/stage0.json b/src/stage0.json new file mode 100644 index 00000000000..3cd97243152 --- /dev/null +++ b/src/stage0.json @@ -0,0 +1,12 @@ +{ + "__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.", + "dist_server": "https://static.rust-lang.org", + "compiler": { + "date": "2021-08-22", + "version": "beta" + }, + "rustfmt": { + "date": "2021-08-26", + "version": "nightly" + } +} diff --git a/src/stage0.txt b/src/stage0.txt deleted file mode 100644 index 6b1507e3650..00000000000 --- a/src/stage0.txt +++ /dev/null @@ -1,42 +0,0 @@ -# This file describes the stage0 compiler that's used to then bootstrap the Rust -# compiler itself. -# -# Currently Rust always bootstraps from the previous stable release, and in our -# train model this means that the master branch bootstraps from beta, beta -# bootstraps from current stable, and stable bootstraps from the previous stable -# release. -# -# If you're looking at this file on the master branch, you'll likely see that -# rustc is configured to `beta`, whereas if you're looking at a source tarball -# for a stable release you'll likely see `1.x.0` for rustc, with the previous -# stable release's version number. `date` is the date where the release we're -# bootstrapping off was released. - -date: 2021-07-29 -rustc: beta - -# We use a nightly rustfmt to format the source because it solves some -# bootstrapping issues with use of new syntax in this repo. If you're looking at -# the beta/stable branch, this key should be omitted, as we don't want to depend -# on rustfmt from nightly there. -rustfmt: nightly-2021-03-25 - -# When making a stable release the process currently looks like: -# -# 1. Produce stable build, upload it to dev-static -# 2. Produce a beta build from the previous stable build, upload to static -# 3. Produce a nightly build from previous beta, upload to static -# 4. Upload stable build to static, publish full release -# -# This means that there's a small window of time (a few days) where artifacts -# are downloaded from dev-static.rust-lang.org instead of static.rust-lang.org. -# In order to ease this transition we have an extra key which is in the -# configuration file below. When uncommented this will instruct the bootstrap.py -# script to download from dev-static.rust-lang.org. -# -# This key is typically commented out at all times. If you're looking at a -# stable release tarball it should *definitely* be commented out. If you're -# looking at a beta source tarball and it's uncommented we'll shortly comment it -# out. - -#dev: 1 diff --git a/src/tools/bump-stage0/Cargo.toml b/src/tools/bump-stage0/Cargo.toml new file mode 100644 index 00000000000..c94331e98e8 --- /dev/null +++ b/src/tools/bump-stage0/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "bump-stage0" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.34" +curl = "0.4.38" +serde = { version = "1.0.125", features = ["derive"] } +serde_json = "1.0.59" +toml = "0.5.7" diff --git a/src/tools/bump-stage0/src/main.rs b/src/tools/bump-stage0/src/main.rs new file mode 100644 index 00000000000..2c673ecccda --- /dev/null +++ b/src/tools/bump-stage0/src/main.rs @@ -0,0 +1,167 @@ +use anyhow::Error; +use curl::easy::Easy; +use std::collections::HashMap; +use std::convert::TryInto; + +const DIST_SERVER: &str = "https://static.rust-lang.org"; + +struct Tool { + channel: Channel, + version: [u16; 3], +} + +impl Tool { + fn new() -> Result { + let channel = match std::fs::read_to_string("src/ci/channel")?.trim() { + "stable" => Channel::Stable, + "beta" => Channel::Beta, + "nightly" => Channel::Nightly, + other => anyhow::bail!("unsupported channel: {}", other), + }; + + // Split "1.42.0" into [1, 42, 0] + let version = std::fs::read_to_string("src/version")? + .trim() + .split('.') + .map(|val| val.parse()) + .collect::, _>>()? + .try_into() + .map_err(|_| anyhow::anyhow!("failed to parse version"))?; + + Ok(Self { channel, version }) + } + + fn update_json(self) -> Result<(), Error> { + std::fs::write( + "src/stage0.json", + format!( + "{}\n", + serde_json::to_string_pretty(&Stage0 { + comment: "Generated by `./x.py run src/tools/bump-stage0`. \ + Run that command again to update the bootstrap compiler.", + dist_server: DIST_SERVER.into(), + compiler: self.detect_compiler()?, + rustfmt: self.detect_rustfmt()?, + })? + ) + )?; + Ok(()) + } + + // Currently Rust always bootstraps from the previous stable release, and in our train model + // this means that the master branch bootstraps from beta, beta bootstraps from current stable, + // and stable bootstraps from the previous stable release. + // + // On the master branch the compiler version is configured to `beta` whereas if you're looking + // at the beta or stable channel you'll likely see `1.x.0` as the version, with the previous + // release's version number. + fn detect_compiler(&self) -> Result { + let channel = match self.channel { + Channel::Stable | Channel::Beta => { + // The 1.XX manifest points to the latest point release of that minor release. + format!("{}.{}", self.version[0], self.version[1] - 1) + } + Channel::Nightly => "beta".to_string(), + }; + + let manifest = fetch_manifest(&channel)?; + Ok(Stage0Toolchain { + date: manifest.date, + version: if self.channel == Channel::Nightly { + "beta".to_string() + } else { + // The version field is like "1.42.0 (abcdef1234 1970-01-01)" + manifest.pkg["rust"] + .version + .split_once(' ') + .expect("invalid version field") + .0 + .to_string() + }, + }) + } + + /// We use a nightly rustfmt to format the source because it solves some bootstrapping issues + /// with use of new syntax in this repo. For the beta/stable channels rustfmt is not provided, + /// as we don't want to depend on rustfmt from nightly there. + fn detect_rustfmt(&self) -> Result, Error> { + if self.channel != Channel::Nightly { + return Ok(None); + } + + let manifest = fetch_manifest("nightly")?; + Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() })) + } +} + +fn main() -> Result<(), Error> { + let tool = Tool::new()?; + tool.update_json()?; + Ok(()) +} + +fn fetch_manifest(channel: &str) -> Result { + Ok(toml::from_slice(&http_get(&format!( + "{}/dist/channel-rust-{}.toml", + DIST_SERVER, channel + ))?)?) +} + +fn http_get(url: &str) -> Result, Error> { + let mut data = Vec::new(); + let mut handle = Easy::new(); + handle.fail_on_error(true)?; + handle.url(url)?; + { + let mut transfer = handle.transfer(); + transfer.write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + })?; + transfer.perform()?; + } + Ok(data) +} + +#[derive(Debug, PartialEq, Eq)] +enum Channel { + Stable, + Beta, + Nightly, +} + +#[derive(Debug, serde::Serialize)] +struct Stage0 { + #[serde(rename = "__comment")] + comment: &'static str, + dist_server: String, + compiler: Stage0Toolchain, + rustfmt: Option, +} + +#[derive(Debug, serde::Serialize)] +struct Stage0Toolchain { + date: String, + version: String, +} + +#[derive(Debug, serde::Deserialize)] +struct Manifest { + date: String, + pkg: HashMap, +} + +#[derive(Debug, serde::Deserialize)] +struct ManifestPackage { + version: String, + target: HashMap, +} + +#[derive(Debug, serde::Deserialize)] +struct ManifestTargetPackage { + available: bool, + url: Option, + hash: Option, + xz_url: Option, + xz_hash: Option, +}