mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

switch stage0.txt to stage0.json and add a tool to generate it

Browse Source
This commit is contained in:
Pietro Albini 2021-08-26 11:26:03 +02:00
parent 33fdb797f5
commit 80b81adc63
No known key found for this signature in database
GPG Key ID: CD76B35F7734769E
13 changed files with 263 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Cargo.lock
View File

@ -220,6 +220,17 @@ dependencies = [
name = "build_helper"
version = "0.1.0"
[[package]]
name = "bump-stage0"
version = "0.1.0"
dependencies = [
"anyhow",
"curl",
"serde",
"serde_json",
"toml",
]
[[package]]
name = "byte-tools"
version = "0.3.1"

1
Cargo.toml
View File

@ -35,6 +35,7 @@ members = [
"src/tools/expand-yaml-anchors",
"src/tools/jsondocck",
"src/tools/html-checker",
"src/tools/bump-stage0",
]
exclude = [

76
src/bootstrap/bootstrap.py
View File

@ -4,6 +4,7 @@ import contextlib
import datetime
import distutils.version
import hashlib
import json
import os
import re
import shutil
@ -176,15 +177,6 @@ def require(cmd, exit=True):
sys.exit(1)
def stage0_data(rust_root):
"""Build a dictionary from stage0.txt"""
nightlies = os.path.join(rust_root, "src/stage0.txt")
with open(nightlies, 'r') as nightlies:
lines = [line.rstrip() for line in nightlies
if not line.startswith("#")]
return dict([line.split(": ", 1) for line in lines if line])
def format_build_time(duration):
"""Return a nicer format for build time
@ -371,13 +363,21 @@ def output(filepath):
os.rename(tmp, filepath)
class Stage0Toolchain:
def __init__(self, stage0_payload):
self.date = stage0_payload["date"]
self.version = stage0_payload["version"]
def channel(self):
return self.version + "-" + self.date
class RustBuild(object):
"""Provide all the methods required to build Rust"""
def __init__(self):
self.date = ''
self.stage0_compiler = None
self.stage0_rustfmt = None
self._download_url = ''
self.rustc_channel = ''
self.rustfmt_channel = ''
self.build = ''
self.build_dir = ''
self.clean = False
@ -401,11 +401,10 @@ class RustBuild(object):
will move all the content to the right place.
"""
if rustc_channel is None:
rustc_channel = self.rustc_channel
rustfmt_channel = self.rustfmt_channel
rustc_channel = self.stage0_compiler.version
bin_root = self.bin_root(stage0)
key = self.date
key = self.stage0_compiler.date
if not stage0:
key += str(self.rustc_commit)
if self.rustc(stage0).startswith(bin_root) and \
@ -444,19 +443,23 @@ class RustBuild(object):
if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
not os.path.exists(self.rustfmt())
or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
or self.program_out_of_date(
self.rustfmt_stamp(),
"" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
)
):
if rustfmt_channel:
if self.stage0_rustfmt is not None:
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
[channel, date] = rustfmt_channel.split('-', 1)
filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
filename = "rustfmt-{}-{}{}".format(
self.stage0_rustfmt.version, self.build, tarball_suffix,
)
self._download_component_helper(
filename, "rustfmt-preview", tarball_suffix, key=date
filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
)
self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.rustfmt_channel)
rustfmt_stamp.write(self.stage0_rustfmt.channel())
# Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
if self.downloading_llvm() and stage0:
@ -517,7 +520,7 @@ class RustBuild(object):
):
if key is None:
if stage0:
key = self.date
key = self.stage0_compiler.date
else:
key = self.rustc_commit
cache_dst = os.path.join(self.build_dir, "cache")
@ -815,7 +818,7 @@ class RustBuild(object):
def rustfmt(self):
"""Return config path for rustfmt"""
if not self.rustfmt_channel:
if self.stage0_rustfmt is None:
return None
return self.program_config('rustfmt')
@ -1039,19 +1042,12 @@ class RustBuild(object):
self.update_submodule(module[0], module[1], recorded_submodules)
print("Submodules updated in %.2f seconds" % (time() - start_time))
def set_normal_environment(self):
def set_dist_environment(self, url):
"""Set download URL for normal environment"""
if 'RUSTUP_DIST_SERVER' in os.environ:
self._download_url = os.environ['RUSTUP_DIST_SERVER']
else:
self._download_url = 'https://static.rust-lang.org'
def set_dev_environment(self):
"""Set download URL for development environment"""
if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
else:
self._download_url = 'https://dev-static.rust-lang.org'
self._download_url = url
def check_vendored_status(self):
"""Check that vendoring is configured properly"""
@ -1160,17 +1156,13 @@ def bootstrap(help_triggered):
build_dir = build.get_toml('build-dir', 'build') or 'build'
build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
data = stage0_data(build.rust_root)
build.date = data['date']
build.rustc_channel = data['rustc']
with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
data = json.load(f)
build.stage0_compiler = Stage0Toolchain(data["compiler"])
if data.get("rustfmt") is not None:
build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
if "rustfmt" in data:
build.rustfmt_channel = data['rustfmt']
if 'dev' in data:
build.set_dev_environment()
else:
build.set_normal_environment()
build.set_dist_environment(data["dist_server"])
build.build = args.build or build.build_triple()
build.update_submodules()

20
src/bootstrap/bootstrap_test.py
View File

@ -13,25 +13,6 @@ from shutil import rmtree
import bootstrap
class Stage0DataTestCase(unittest.TestCase):
"""Test Case for stage0_data"""
def setUp(self):
self.rust_root = tempfile.mkdtemp()
os.mkdir(os.path.join(self.rust_root, "src"))
with open(os.path.join(self.rust_root, "src",
"stage0.txt"), "w") as stage0:
stage0.write("#ignore\n\ndate: 2017-06-15\nrustc: beta\ncargo: beta\nrustfmt: beta")
def tearDown(self):
rmtree(self.rust_root)
def test_stage0_data(self):
"""Extract data from stage0.txt"""
expected = {"date": "2017-06-15", "rustc": "beta", "cargo": "beta", "rustfmt": "beta"}
data = bootstrap.stage0_data(self.rust_root)
self.assertDictEqual(data, expected)
class VerifyTestCase(unittest.TestCase):
"""Test Case for verify"""
def setUp(self):
@ -99,7 +80,6 @@ if __name__ == '__main__':
TEST_LOADER = unittest.TestLoader()
SUITE.addTest(doctest.DocTestSuite(bootstrap))
SUITE.addTests([
TEST_LOADER.loadTestsFromTestCase(Stage0DataTestCase),
TEST_LOADER.loadTestsFromTestCase(VerifyTestCase),
TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)])

2
src/bootstrap/builder.rs
View File

@ -523,7 +523,7 @@ macro_rules! describe {
install::Src,
install::Rustc
),
Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest),
Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest, run::BumpStage0),
}
}

2
src/bootstrap/lib.rs
View File

@ -31,7 +31,7 @@
//! When you execute `x.py build`, the steps executed are:
//!
//! * First, the python script is run. This will automatically download the
//! stage0 rustc and cargo according to `src/stage0.txt`, or use the cached
//! stage0 rustc and cargo according to `src/stage0.json`, or use the cached
//! versions if they're available. These are then used to compile rustbuild
//! itself (using Cargo). Finally, control is then transferred to rustbuild.
//!

21
src/bootstrap/run.rs
View File

@ -82,3 +82,24 @@ fn run(self, builder: &Builder<'_>) {
builder.run(&mut cmd);
}
}
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
pub struct BumpStage0;
impl Step for BumpStage0 {
type Output = ();
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/bump-stage0")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(BumpStage0);
}
fn run(self, builder: &Builder<'_>) -> Self::Output {
let mut cmd = builder.tool_cmd(Tool::BumpStage0);
builder.run(&mut cmd);
}
}

12
src/bootstrap/sanity.rs
View File

@ -15,7 +15,7 @@
use std::path::PathBuf;
use std::process::Command;
use build_helper::{output, t};
use build_helper::output;
use crate::cache::INTERNER;
use crate::config::Target;
@ -227,14 +227,4 @@ pub fn check(build: &mut Build) {
if let Some(ref s) = build.config.ccache {
cmd_finder.must_have(s);
}
if build.config.channel == "stable" {
let stage0 = t!(fs::read_to_string(build.src.join("src/stage0.txt")));
if stage0.contains("\ndev:") {
panic!(
"bootstrapping from a dev compiler in a stable release, but \
should only be bootstrapping from a released compiler!"
);
}
}
}

1
src/bootstrap/tool.rs
View File

@ -377,6 +377,7 @@ fn run(self, builder: &Builder<'_>) -> PathBuf {
LintDocs, "src/tools/lint-docs", "lint-docs";
JsonDocCk, "src/tools/jsondocck", "jsondocck";
HtmlChecker, "src/tools/html-checker", "html-checker";
BumpStage0, "src/tools/bump-stage0", "bump-stage0";
);
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]

12
src/stage0.json Normal file
View File

@ -0,0 +1,12 @@
{
"__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.",
"dist_server": "https://static.rust-lang.org",
"compiler": {
"date": "2021-08-22",
"version": "beta"
},
"rustfmt": {
"date": "2021-08-26",
"version": "nightly"
}
}

42
src/stage0.txt
View File

@ -1,42 +0,0 @@
# This file describes the stage0 compiler that's used to then bootstrap the Rust
# compiler itself.
#
# Currently Rust always bootstraps from the previous stable release, and in our
# train model this means that the master branch bootstraps from beta, beta
# bootstraps from current stable, and stable bootstraps from the previous stable
# release.
#
# If you're looking at this file on the master branch, you'll likely see that
# rustc is configured to `beta`, whereas if you're looking at a source tarball
# for a stable release you'll likely see `1.x.0` for rustc, with the previous
# stable release's version number. `date` is the date where the release we're
# bootstrapping off was released.
date: 2021-07-29
rustc: beta
# We use a nightly rustfmt to format the source because it solves some
# bootstrapping issues with use of new syntax in this repo. If you're looking at
# the beta/stable branch, this key should be omitted, as we don't want to depend
# on rustfmt from nightly there.
rustfmt: nightly-2021-03-25
# When making a stable release the process currently looks like:
#
# 1. Produce stable build, upload it to dev-static
# 2. Produce a beta build from the previous stable build, upload to static
# 3. Produce a nightly build from previous beta, upload to static
# 4. Upload stable build to static, publish full release
#
# This means that there's a small window of time (a few days) where artifacts
# are downloaded from dev-static.rust-lang.org instead of static.rust-lang.org.
# In order to ease this transition we have an extra key which is in the
# configuration file below. When uncommented this will instruct the bootstrap.py
# script to download from dev-static.rust-lang.org.
#
# This key is typically commented out at all times. If you're looking at a
# stable release tarball it should *definitely* be commented out. If you're
# looking at a beta source tarball and it's uncommented we'll shortly comment it
# out.
#dev: 1

13
src/tools/bump-stage0/Cargo.toml Normal file
View File

@ -0,0 +1,13 @@
[package]
name = "bump-stage0"
version = "0.1.0"
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = "1.0.34"
curl = "0.4.38"
serde = { version = "1.0.125", features = ["derive"] }
serde_json = "1.0.59"
toml = "0.5.7"

167
src/tools/bump-stage0/src/main.rs Normal file
View File

@ -0,0 +1,167 @@
use anyhow::Error;
use curl::easy::Easy;
use std::collections::HashMap;
use std::convert::TryInto;
const DIST_SERVER: &str = "https://static.rust-lang.org";
struct Tool {
channel: Channel,
version: [u16; 3],
}
impl Tool {
fn new() -> Result<Self, Error> {
let channel = match std::fs::read_to_string("src/ci/channel")?.trim() {
"stable" => Channel::Stable,
"beta" => Channel::Beta,
"nightly" => Channel::Nightly,
other => anyhow::bail!("unsupported channel: {}", other),
};
// Split "1.42.0" into [1, 42, 0]
let version = std::fs::read_to_string("src/version")?
.trim()
.split('.')
.map(|val| val.parse())
.collect::<Result<Vec<_>, _>>()?
.try_into()
.map_err(|_| anyhow::anyhow!("failed to parse version"))?;
Ok(Self { channel, version })
}
fn update_json(self) -> Result<(), Error> {
std::fs::write(
"src/stage0.json",
format!(
"{}\n",
serde_json::to_string_pretty(&Stage0 {
comment: "Generated by `./x.py run src/tools/bump-stage0`. \
Run that command again to update the bootstrap compiler.",
dist_server: DIST_SERVER.into(),
compiler: self.detect_compiler()?,
rustfmt: self.detect_rustfmt()?,
})?
)
)?;
Ok(())
}
// Currently Rust always bootstraps from the previous stable release, and in our train model
// this means that the master branch bootstraps from beta, beta bootstraps from current stable,
// and stable bootstraps from the previous stable release.
//
// On the master branch the compiler version is configured to `beta` whereas if you're looking
// at the beta or stable channel you'll likely see `1.x.0` as the version, with the previous
// release's version number.
fn detect_compiler(&self) -> Result<Stage0Toolchain, Error> {
let channel = match self.channel {
Channel::Stable | Channel::Beta => {
// The 1.XX manifest points to the latest point release of that minor release.
format!("{}.{}", self.version[0], self.version[1] - 1)
}
Channel::Nightly => "beta".to_string(),
};
let manifest = fetch_manifest(&channel)?;
Ok(Stage0Toolchain {
date: manifest.date,
version: if self.channel == Channel::Nightly {
"beta".to_string()
} else {
// The version field is like "1.42.0 (abcdef1234 1970-01-01)"
manifest.pkg["rust"]
.version
.split_once(' ')
.expect("invalid version field")
.0
.to_string()
},
})
}
/// We use a nightly rustfmt to format the source because it solves some bootstrapping issues
/// with use of new syntax in this repo. For the beta/stable channels rustfmt is not provided,
/// as we don't want to depend on rustfmt from nightly there.
fn detect_rustfmt(&self) -> Result<Option<Stage0Toolchain>, Error> {
if self.channel != Channel::Nightly {
return Ok(None);
}
let manifest = fetch_manifest("nightly")?;
Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() }))
}
}
fn main() -> Result<(), Error> {
let tool = Tool::new()?;
tool.update_json()?;
Ok(())
}
fn fetch_manifest(channel: &str) -> Result<Manifest, Error> {
Ok(toml::from_slice(&http_get(&format!(
"{}/dist/channel-rust-{}.toml",
DIST_SERVER, channel
))?)?)
}
fn http_get(url: &str) -> Result<Vec<u8>, Error> {
let mut data = Vec::new();
let mut handle = Easy::new();
handle.fail_on_error(true)?;
handle.url(url)?;
{
let mut transfer = handle.transfer();
transfer.write_function(|new_data| {
data.extend_from_slice(new_data);
Ok(new_data.len())
})?;
transfer.perform()?;
}
Ok(data)
}
#[derive(Debug, PartialEq, Eq)]
enum Channel {
Stable,
Beta,
Nightly,
}
#[derive(Debug, serde::Serialize)]
struct Stage0 {
#[serde(rename = "__comment")]
comment: &'static str,
dist_server: String,
compiler: Stage0Toolchain,
rustfmt: Option<Stage0Toolchain>,
}
#[derive(Debug, serde::Serialize)]
struct Stage0Toolchain {
date: String,
version: String,
}
#[derive(Debug, serde::Deserialize)]
struct Manifest {
date: String,
pkg: HashMap<String, ManifestPackage>,
}
#[derive(Debug, serde::Deserialize)]
struct ManifestPackage {
version: String,
target: HashMap<String, ManifestTargetPackage>,
}
#[derive(Debug, serde::Deserialize)]
struct ManifestTargetPackage {
available: bool,
url: Option<String>,
hash: Option<String>,
xz_url: Option<String>,
xz_hash: Option<String>,
}