mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use shared stage0 parser from build_helper

Browse Source 
Signed-off-by: onur-ozkan <work@onurozkan.dev>
This commit is contained in:
onur-ozkan 2024-05-09 18:18:01 +03:00
parent f2d50b6d36
commit b46c3f279d
7 changed files with 120 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock
View File

@ -380,10 +380,10 @@ name = "bump-stage0"
version = "0.1.0"
dependencies = [
"anyhow",
"build_helper",
"curl",
"indexmap",
"serde",
"serde_json",
"toml 0.5.11",
]

38
src/bootstrap/src/core/config/config.rs
View File

@ -181,7 +181,7 @@ pub struct Config {
pub test_compare_mode: bool,
pub color: Color,
pub patch_binaries_for_nix: Option<bool>,
pub stage0_metadata: Stage0Metadata,
pub stage0_metadata: build_helper::stage0_parser::Stage0,
pub android_ndk: Option<PathBuf>,
/// Whether to use the `c` feature of the `compiler_builtins` crate.
pub optimized_compiler_builtins: bool,
@ -350,34 +350,6 @@ pub struct Config {
pub paths: Vec<PathBuf>,
}
#[derive(Default, Deserialize, Clone)]
pub struct Stage0Metadata {
pub compiler: CompilerMetadata,
pub config: Stage0Config,
pub checksums_sha256: HashMap<String, String>,
pub rustfmt: Option<RustfmtMetadata>,
}
#[derive(Default, Deserialize, Clone)]
pub struct CompilerMetadata {
pub date: String,
pub version: String,
}
#[derive(Default, Deserialize, Clone)]
pub struct Stage0Config {
pub dist_server: String,
pub artifacts_server: String,
pub artifacts_with_llvm_assertions_server: String,
pub git_merge_commit_email: String,
pub git_repository: String,
pub nightly_branch: String,
}
#[derive(Default, Deserialize, Clone)]
pub struct RustfmtMetadata {
pub date: String,
pub version: String,
}
#[derive(Clone, Debug, Default)]
pub enum RustfmtState {
SystemToolchain(PathBuf),
@ -1296,13 +1268,13 @@ pub(crate) fn parse_inner(args: &[String], get_toml: impl Fn(&Path) -> TomlConfi
Some(p) => PathBuf::from(p),
None => git_root,
};
// If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when,
// If this doesn't have at least `stage0`, we guessed wrong. This can happen when,
// for example, the build directory is inside of another unrelated git directory.
// In that case keep the original `CARGO_MANIFEST_DIR` handling.
//
// NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside
// the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1.
if git_root.join("src").join("stage0.json").exists() {
if git_root.join("src").join("stage0").exists() {
config.src = git_root;
}
} else {
@ -1320,9 +1292,7 @@ pub(crate) fn parse_inner(args: &[String], get_toml: impl Fn(&Path) -> TomlConfi
.to_path_buf();
}
let stage0_json = t!(std::fs::read(config.src.join("src").join("stage0.json")));
config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
config.stage0_metadata = build_helper::stage0_parser::parse_stage0_file();
// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory.
let toml_path = flags

8
src/bootstrap/src/core/download.rs
View File

@ -9,9 +9,9 @@
};
use build_helper::ci::CiEnv;
use build_helper::stage0_parser::VersionMetadata;
use xz2::bufread::XzDecoder;
use crate::core::config::RustfmtMetadata;
use crate::utils::helpers::{check_run, exe, program_out_of_date};
use crate::{core::build_steps::llvm::detect_llvm_sha, utils::helpers::hex_encode};
use crate::{t, Config};
@ -408,7 +408,7 @@ pub(crate) fn download_clippy(&self) -> PathBuf {
/// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't
/// reuse target directories or artifacts
pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
let channel = format!("{version}-{date}");
let host = self.build;
@ -606,7 +606,7 @@ fn download_component(
DownloadSource::Dist => {
let dist_server = env::var("RUSTUP_DIST_SERVER")
.unwrap_or(self.stage0_metadata.config.dist_server.to_string());
// NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
// NOTE: make `dist` part of the URL because that's how it's stored in src/stage0
(dist_server, format!("dist/{key}/{filename}"), true)
}
};
@ -616,7 +616,7 @@ fn download_component(
// this on each and every nightly ...
let checksum = if should_verify {
let error = format!(
"src/stage0.json doesn't contain a checksum for {url}. \
"src/stage0 doesn't contain a checksum for {url}. \
Pre-built artifacts might not be available for this \
target at this time, see https://doc.rust-lang.org/nightly\
/rustc/platform-support.html for more information."

1
src/tools/build_helper/src/lib.rs
View File

@ -2,3 +2,4 @@
pub mod git;
pub mod metrics;
pub mod util;
pub mod stage0_parser;

76
src/tools/build_helper/src/stage0_parser.rs Normal file
View File

@ -0,0 +1,76 @@
use std::collections::BTreeMap;
#[derive(Default, Clone)]
pub struct Stage0 {
pub compiler: VersionMetadata,
pub rustfmt: Option<VersionMetadata>,
pub config: Stage0Config,
pub checksums_sha256: BTreeMap<String, String>,
}
#[derive(Default, Clone)]
pub struct VersionMetadata {
pub date: String,
pub version: String,
}
#[derive(Default, Clone)]
pub struct Stage0Config {
pub dist_server: String,
pub artifacts_server: String,
pub artifacts_with_llvm_assertions_server: String,
pub git_merge_commit_email: String,
pub git_repository: String,
pub nightly_branch: String,
}
pub fn parse_stage0_file() -> Stage0 {
let stage0_content = include_str!("../../../stage0");
let mut stage0 = Stage0::default();
for line in stage0_content.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
// Ignore comments
if line.starts_with('#') {
continue;
}
let (key, value) = line.split_once('=').unwrap();
match key {
"dist_server" => stage0.config.dist_server = value.to_owned(),
"artifacts_server" => stage0.config.artifacts_server = value.to_owned(),
"artifacts_with_llvm_assertions_server" => {
stage0.config.artifacts_with_llvm_assertions_server = value.to_owned()
}
"git_merge_commit_email" => stage0.config.git_merge_commit_email = value.to_owned(),
"git_repository" => stage0.config.git_repository = value.to_owned(),
"nightly_branch" => stage0.config.nightly_branch = value.to_owned(),
"compiler_date" => stage0.compiler.date = value.to_owned(),
"compiler_version" => stage0.compiler.version = value.to_owned(),
"rustfmt_date" => {
stage0.rustfmt.get_or_insert(VersionMetadata::default()).date = value.to_owned();
}
"rustfmt_version" => {
stage0.rustfmt.get_or_insert(VersionMetadata::default()).version = value.to_owned();
}
dist if dist.starts_with("dist") => {
stage0.checksums_sha256.insert(key.to_owned(), value.to_owned());
}
unsupported => {
println!("'{unsupported}' field is not supported.");
}
}
}
stage0
}

2
src/tools/bump-stage0/Cargo.toml
View File

@ -7,8 +7,8 @@ edition = "2021"
[dependencies]
anyhow = "1.0.34"
build_helper = { path = "../build_helper" }
curl = "0.4.38"
indexmap = { version = "2.0.0", features = ["serde"] }
serde = { version = "1.0.125", features = ["derive"] }
serde_json = { version = "1.0.59", features = ["preserve_order"] }
toml = "0.5.7"

134
src/tools/bump-stage0/src/main.rs
View File

@ -1,4 +1,7 @@
#![deny(unused_variables)]
use anyhow::{Context, Error};
use build_helper::stage0_parser::{parse_stage0_file, Stage0Config, VersionMetadata};
use curl::easy::Easy;
use indexmap::IndexMap;
use std::collections::HashMap;
@ -8,7 +11,7 @@
const RUSTFMT_COMPONENTS: &[&str] = &["rustfmt-preview", "rustc"];
struct Tool {
config: Config,
config: Stage0Config,
channel: Channel,
date: Option<String>,
@ -34,73 +37,9 @@ fn new(date: Option<String>) -> Result<Self, Error> {
.try_into()
.map_err(|_| anyhow::anyhow!("failed to parse version"))?;
// let existing: Stage0 = serde_json::from_slice(&std::fs::read(PATH)?)?;
let existing = Self::parse_stage0_file()?;
let existing = parse_stage0_file();
Ok(Self {
channel,
version,
date,
config: existing.config,
checksums: IndexMap::new(),
})
}
fn parse_stage0_file() -> Result<Stage0, Error> {
let stage0_content = include_str!("../../../stage0");
let mut stage0 = Stage0::default();
for line in stage0_content.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
// Ignore comments
if line.starts_with('#') {
continue;
}
let key_value: Vec<&str> = line.splitn(2, '=').collect();
let (key, value) = (*key_value.get(0).unwrap(), *key_value.get(1).unwrap());
match key {
"dist_server" => stage0.config.dist_server = value.to_owned(),
"artifacts_server"
| "artifacts_with_llvm_assertions_server"
| "git_merge_commit_email"
| "git_repository"
| "nightly_branch" => {
stage0.config.other.insert(key.to_owned(), value.to_owned());
}
"compiler_date" => stage0.compiler.date = value.to_owned(),
"compiler_version" => stage0.compiler.version = value.to_owned(),
"rustfmt_date" => {
let mut rustfmt = stage0.rustfmt.unwrap_or(Stage0Toolchain::default());
rustfmt.date = value.to_owned();
stage0.rustfmt = Some(rustfmt);
}
"rustfmt_version" => {
let mut rustfmt = stage0.rustfmt.unwrap_or(Stage0Toolchain::default());
rustfmt.version = value.to_owned();
stage0.rustfmt = Some(rustfmt);
}
dist if dist.starts_with("dist") => {
stage0.checksums_sha256.insert(key.to_owned(), value.to_owned());
}
unsupported => {
println!("'{unsupported}' field is not supported.");
}
}
}
Ok(stage0)
Ok(Self { channel, version, date, config: existing.config, checksums: IndexMap::new() })
}
fn update_stage0_file(mut self) -> Result<(), Error> {
@ -115,11 +54,25 @@ fn update_stage0_file(mut self) -> Result<(), Error> {
"#;
let mut file_content = HEADER.to_owned();
file_content.push_str(&format!("\ndist_server={}", self.config.dist_server));
for (key, value) in &self.config.other {
file_content.push_str(&format!("\n{}={}", key, value.as_str()));
}
let Stage0Config {
dist_server,
artifacts_server,
artifacts_with_llvm_assertions_server,
git_merge_commit_email,
git_repository,
nightly_branch,
} = &self.config;
file_content.push_str(&format!("\ndist_server={}", dist_server));
file_content.push_str(&format!("\nartifacts_server={}", artifacts_server));
file_content.push_str(&format!(
"\nartifacts_with_llvm_assertions_server={}",
artifacts_with_llvm_assertions_server
));
file_content.push_str(&format!("\ngit_merge_commit_email={}", git_merge_commit_email));
file_content.push_str(&format!("\ngit_repository={}", git_repository));
file_content.push_str(&format!("\nnightly_branch={}", nightly_branch));
file_content.push_str("\n");
@ -149,7 +102,7 @@ fn update_stage0_file(mut self) -> Result<(), Error> {
// On the master branch the compiler version is configured to `beta` whereas if you're looking
// at the beta or stable channel you'll likely see `1.x.0` as the version, with the previous
// release's version number.
fn detect_compiler(&mut self) -> Result<Stage0Toolchain, Error> {
fn detect_compiler(&mut self) -> Result<VersionMetadata, Error> {
let channel = match self.channel {
Channel::Stable | Channel::Beta => {
// The 1.XX manifest points to the latest point release of that minor release.
@ -160,7 +113,7 @@ fn detect_compiler(&mut self) -> Result<Stage0Toolchain, Error> {
let manifest = fetch_manifest(&self.config, &channel, self.date.as_deref())?;
self.collect_checksums(&manifest, COMPILER_COMPONENTS)?;
Ok(Stage0Toolchain {
Ok(VersionMetadata {
date: manifest.date,
version: if self.channel == Channel::Nightly {
"beta".to_string()
@ -179,14 +132,14 @@ fn detect_compiler(&mut self) -> Result<Stage0Toolchain, Error> {
/// We use a nightly rustfmt to format the source because it solves some bootstrapping issues
/// with use of new syntax in this repo. For the beta/stable channels rustfmt is not provided,
/// as we don't want to depend on rustfmt from nightly there.
fn detect_rustfmt(&mut self) -> Result<Option<Stage0Toolchain>, Error> {
fn detect_rustfmt(&mut self) -> Result<Option<VersionMetadata>, Error> {
if self.channel != Channel::Nightly {
return Ok(None);
}
let manifest = fetch_manifest(&self.config, "nightly", self.date.as_deref())?;
self.collect_checksums(&manifest, RUSTFMT_COMPONENTS)?;
Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() }))
Ok(Some(VersionMetadata { date: manifest.date, version: "nightly".into() }))
}
fn collect_checksums(&mut self, manifest: &Manifest, components: &[&str]) -> Result<(), Error> {
@ -220,7 +173,11 @@ fn main() -> Result<(), Error> {
Ok(())
}
fn fetch_manifest(config: &Config, channel: &str, date: Option<&str>) -> Result<Manifest, Error> {
fn fetch_manifest(
config: &Stage0Config,
channel: &str,
date: Option<&str>,
) -> Result<Manifest, Error> {
let url = if let Some(date) = date {
format!("{}/dist/{}/channel-rust-{}.toml", config.dist_server, date, channel)
} else {
@ -253,31 +210,6 @@ enum Channel {
Nightly,
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
struct Stage0 {
config: Config,
compiler: Stage0Toolchain,
rustfmt: Option<Stage0Toolchain>,
checksums_sha256: IndexMap<String, String>,
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
struct Config {
dist_server: String,
/// There are other fields in the configuration, which will be read by src/bootstrap or other
/// tools consuming stage0 file. To avoid the need to update bump-stage0 every time a new field
/// is added, we collect all the fields in `IndexMap<String, String>` and serialize them back with the
/// same order and structure they were deserialized in.
#[serde(flatten)]
other: IndexMap<String, String>,
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
struct Stage0Toolchain {
date: String,
version: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct Manifest {
date: String,