diff --git a/Cargo.lock b/Cargo.lock index c034cbae9912..aeefa89a608d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,10 +380,10 @@ name = "bump-stage0" version = "0.1.0" dependencies = [ "anyhow", + "build_helper", "curl", "indexmap", "serde", - "serde_json", "toml 0.5.11", ] diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index ed45bc30362a..5550b2047f57 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -181,7 +181,7 @@ pub struct Config { pub test_compare_mode: bool, pub color: Color, pub patch_binaries_for_nix: Option, - pub stage0_metadata: Stage0Metadata, + pub stage0_metadata: build_helper::stage0_parser::Stage0, pub android_ndk: Option, /// Whether to use the `c` feature of the `compiler_builtins` crate. pub optimized_compiler_builtins: bool, @@ -350,34 +350,6 @@ pub struct Config { pub paths: Vec, } -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Metadata { - pub compiler: CompilerMetadata, - pub config: Stage0Config, - pub checksums_sha256: HashMap, - pub rustfmt: Option, -} -#[derive(Default, Deserialize, Clone)] -pub struct CompilerMetadata { - pub date: String, - pub version: String, -} - -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Config { - pub dist_server: String, - pub artifacts_server: String, - pub artifacts_with_llvm_assertions_server: String, - pub git_merge_commit_email: String, - pub git_repository: String, - pub nightly_branch: String, -} -#[derive(Default, Deserialize, Clone)] -pub struct RustfmtMetadata { - pub date: String, - pub version: String, -} - #[derive(Clone, Debug, Default)] pub enum RustfmtState { SystemToolchain(PathBuf), @@ -1296,13 +1268,13 @@ impl Config { Some(p) => PathBuf::from(p), None => git_root, }; - // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, + // If this doesn't have at least `stage0`, we guessed wrong. This can happen when, // for example, the build directory is inside of another unrelated git directory. // In that case keep the original `CARGO_MANIFEST_DIR` handling. // // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if git_root.join("src").join("stage0.json").exists() { + if git_root.join("src").join("stage0").exists() { config.src = git_root; } } else { @@ -1320,9 +1292,7 @@ impl Config { .to_path_buf(); } - let stage0_json = t!(std::fs::read(config.src.join("src").join("stage0.json"))); - - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + config.stage0_metadata = build_helper::stage0_parser::parse_stage0_file(); // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. let toml_path = flags diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index 75e0f646da69..a074d53aa36e 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -9,9 +9,9 @@ use std::{ }; use build_helper::ci::CiEnv; +use build_helper::stage0_parser::VersionMetadata; use xz2::bufread::XzDecoder; -use crate::core::config::RustfmtMetadata; use crate::utils::helpers::{check_run, exe, program_out_of_date}; use crate::{core::build_steps::llvm::detect_llvm_sha, utils::helpers::hex_encode}; use crate::{t, Config}; @@ -408,7 +408,7 @@ impl Config { /// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't /// reuse target directories or artifacts pub(crate) fn maybe_download_rustfmt(&self) -> Option { - let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; + let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; let channel = format!("{version}-{date}"); let host = self.build; @@ -606,7 +606,7 @@ impl Config { DownloadSource::Dist => { let dist_server = env::var("RUSTUP_DIST_SERVER") .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); - // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0 (dist_server, format!("dist/{key}/{filename}"), true) } }; @@ -616,7 +616,7 @@ impl Config { // this on each and every nightly ... let checksum = if should_verify { let error = format!( - "src/stage0.json doesn't contain a checksum for {url}. \ + "src/stage0 doesn't contain a checksum for {url}. \ Pre-built artifacts might not be available for this \ target at this time, see https://doc.rust-lang.org/nightly\ /rustc/platform-support.html for more information." diff --git a/src/tools/build_helper/src/lib.rs b/src/tools/build_helper/src/lib.rs index 575f3677155e..6a4e86eb1dfe 100644 --- a/src/tools/build_helper/src/lib.rs +++ b/src/tools/build_helper/src/lib.rs @@ -2,3 +2,4 @@ pub mod ci; pub mod git; pub mod metrics; pub mod util; +pub mod stage0_parser; diff --git a/src/tools/build_helper/src/stage0_parser.rs b/src/tools/build_helper/src/stage0_parser.rs new file mode 100644 index 000000000000..ff05b1169895 --- /dev/null +++ b/src/tools/build_helper/src/stage0_parser.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeMap; + +#[derive(Default, Clone)] +pub struct Stage0 { + pub compiler: VersionMetadata, + pub rustfmt: Option, + pub config: Stage0Config, + pub checksums_sha256: BTreeMap, +} + +#[derive(Default, Clone)] +pub struct VersionMetadata { + pub date: String, + pub version: String, +} + +#[derive(Default, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub git_repository: String, + pub nightly_branch: String, +} + +pub fn parse_stage0_file() -> Stage0 { + let stage0_content = include_str!("../../../stage0"); + + let mut stage0 = Stage0::default(); + for line in stage0_content.lines() { + let line = line.trim(); + + if line.is_empty() { + continue; + } + + // Ignore comments + if line.starts_with('#') { + continue; + } + + let (key, value) = line.split_once('=').unwrap(); + + match key { + "dist_server" => stage0.config.dist_server = value.to_owned(), + "artifacts_server" => stage0.config.artifacts_server = value.to_owned(), + "artifacts_with_llvm_assertions_server" => { + stage0.config.artifacts_with_llvm_assertions_server = value.to_owned() + } + "git_merge_commit_email" => stage0.config.git_merge_commit_email = value.to_owned(), + "git_repository" => stage0.config.git_repository = value.to_owned(), + "nightly_branch" => stage0.config.nightly_branch = value.to_owned(), + + "compiler_date" => stage0.compiler.date = value.to_owned(), + "compiler_version" => stage0.compiler.version = value.to_owned(), + + "rustfmt_date" => { + stage0.rustfmt.get_or_insert(VersionMetadata::default()).date = value.to_owned(); + } + "rustfmt_version" => { + stage0.rustfmt.get_or_insert(VersionMetadata::default()).version = value.to_owned(); + } + + dist if dist.starts_with("dist") => { + stage0.checksums_sha256.insert(key.to_owned(), value.to_owned()); + } + + unsupported => { + println!("'{unsupported}' field is not supported."); + } + } + } + + stage0 +} diff --git a/src/tools/bump-stage0/Cargo.toml b/src/tools/bump-stage0/Cargo.toml index 4680a7ab6610..de5d821133d5 100644 --- a/src/tools/bump-stage0/Cargo.toml +++ b/src/tools/bump-stage0/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] anyhow = "1.0.34" +build_helper = { path = "../build_helper" } curl = "0.4.38" indexmap = { version = "2.0.0", features = ["serde"] } serde = { version = "1.0.125", features = ["derive"] } -serde_json = { version = "1.0.59", features = ["preserve_order"] } toml = "0.5.7" diff --git a/src/tools/bump-stage0/src/main.rs b/src/tools/bump-stage0/src/main.rs index 1999859ff988..f9c1699eb4ce 100644 --- a/src/tools/bump-stage0/src/main.rs +++ b/src/tools/bump-stage0/src/main.rs @@ -1,4 +1,7 @@ +#![deny(unused_variables)] + use anyhow::{Context, Error}; +use build_helper::stage0_parser::{parse_stage0_file, Stage0Config, VersionMetadata}; use curl::easy::Easy; use indexmap::IndexMap; use std::collections::HashMap; @@ -8,7 +11,7 @@ const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo", "clippy-pre const RUSTFMT_COMPONENTS: &[&str] = &["rustfmt-preview", "rustc"]; struct Tool { - config: Config, + config: Stage0Config, channel: Channel, date: Option, @@ -34,73 +37,9 @@ impl Tool { .try_into() .map_err(|_| anyhow::anyhow!("failed to parse version"))?; - // let existing: Stage0 = serde_json::from_slice(&std::fs::read(PATH)?)?; - let existing = Self::parse_stage0_file()?; + let existing = parse_stage0_file(); - Ok(Self { - channel, - version, - date, - config: existing.config, - checksums: IndexMap::new(), - }) - } - - fn parse_stage0_file() -> Result { - let stage0_content = include_str!("../../../stage0"); - - let mut stage0 = Stage0::default(); - - for line in stage0_content.lines() { - let line = line.trim(); - - if line.is_empty() { - continue; - } - - // Ignore comments - if line.starts_with('#') { - continue; - } - - let key_value: Vec<&str> = line.splitn(2, '=').collect(); - let (key, value) = (*key_value.get(0).unwrap(), *key_value.get(1).unwrap()); - - match key { - "dist_server" => stage0.config.dist_server = value.to_owned(), - "artifacts_server" - | "artifacts_with_llvm_assertions_server" - | "git_merge_commit_email" - | "git_repository" - | "nightly_branch" => { - stage0.config.other.insert(key.to_owned(), value.to_owned()); - } - - "compiler_date" => stage0.compiler.date = value.to_owned(), - "compiler_version" => stage0.compiler.version = value.to_owned(), - - "rustfmt_date" => { - let mut rustfmt = stage0.rustfmt.unwrap_or(Stage0Toolchain::default()); - rustfmt.date = value.to_owned(); - stage0.rustfmt = Some(rustfmt); - } - "rustfmt_version" => { - let mut rustfmt = stage0.rustfmt.unwrap_or(Stage0Toolchain::default()); - rustfmt.version = value.to_owned(); - stage0.rustfmt = Some(rustfmt); - } - - dist if dist.starts_with("dist") => { - stage0.checksums_sha256.insert(key.to_owned(), value.to_owned()); - } - - unsupported => { - println!("'{unsupported}' field is not supported."); - } - } - } - - Ok(stage0) + Ok(Self { channel, version, date, config: existing.config, checksums: IndexMap::new() }) } fn update_stage0_file(mut self) -> Result<(), Error> { @@ -115,11 +54,25 @@ impl Tool { "#; let mut file_content = HEADER.to_owned(); - file_content.push_str(&format!("\ndist_server={}", self.config.dist_server)); - for (key, value) in &self.config.other { - file_content.push_str(&format!("\n{}={}", key, value.as_str())); - } + let Stage0Config { + dist_server, + artifacts_server, + artifacts_with_llvm_assertions_server, + git_merge_commit_email, + git_repository, + nightly_branch, + } = &self.config; + + file_content.push_str(&format!("\ndist_server={}", dist_server)); + file_content.push_str(&format!("\nartifacts_server={}", artifacts_server)); + file_content.push_str(&format!( + "\nartifacts_with_llvm_assertions_server={}", + artifacts_with_llvm_assertions_server + )); + file_content.push_str(&format!("\ngit_merge_commit_email={}", git_merge_commit_email)); + file_content.push_str(&format!("\ngit_repository={}", git_repository)); + file_content.push_str(&format!("\nnightly_branch={}", nightly_branch)); file_content.push_str("\n"); @@ -149,7 +102,7 @@ impl Tool { // On the master branch the compiler version is configured to `beta` whereas if you're looking // at the beta or stable channel you'll likely see `1.x.0` as the version, with the previous // release's version number. - fn detect_compiler(&mut self) -> Result { + fn detect_compiler(&mut self) -> Result { let channel = match self.channel { Channel::Stable | Channel::Beta => { // The 1.XX manifest points to the latest point release of that minor release. @@ -160,7 +113,7 @@ impl Tool { let manifest = fetch_manifest(&self.config, &channel, self.date.as_deref())?; self.collect_checksums(&manifest, COMPILER_COMPONENTS)?; - Ok(Stage0Toolchain { + Ok(VersionMetadata { date: manifest.date, version: if self.channel == Channel::Nightly { "beta".to_string() @@ -179,14 +132,14 @@ impl Tool { /// We use a nightly rustfmt to format the source because it solves some bootstrapping issues /// with use of new syntax in this repo. For the beta/stable channels rustfmt is not provided, /// as we don't want to depend on rustfmt from nightly there. - fn detect_rustfmt(&mut self) -> Result, Error> { + fn detect_rustfmt(&mut self) -> Result, Error> { if self.channel != Channel::Nightly { return Ok(None); } let manifest = fetch_manifest(&self.config, "nightly", self.date.as_deref())?; self.collect_checksums(&manifest, RUSTFMT_COMPONENTS)?; - Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() })) + Ok(Some(VersionMetadata { date: manifest.date, version: "nightly".into() })) } fn collect_checksums(&mut self, manifest: &Manifest, components: &[&str]) -> Result<(), Error> { @@ -220,7 +173,11 @@ fn main() -> Result<(), Error> { Ok(()) } -fn fetch_manifest(config: &Config, channel: &str, date: Option<&str>) -> Result { +fn fetch_manifest( + config: &Stage0Config, + channel: &str, + date: Option<&str>, +) -> Result { let url = if let Some(date) = date { format!("{}/dist/{}/channel-rust-{}.toml", config.dist_server, date, channel) } else { @@ -253,31 +210,6 @@ enum Channel { Nightly, } -#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] -struct Stage0 { - config: Config, - compiler: Stage0Toolchain, - rustfmt: Option, - checksums_sha256: IndexMap, -} - -#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] -struct Config { - dist_server: String, - /// There are other fields in the configuration, which will be read by src/bootstrap or other - /// tools consuming stage0 file. To avoid the need to update bump-stage0 every time a new field - /// is added, we collect all the fields in `IndexMap` and serialize them back with the - /// same order and structure they were deserialized in. - #[serde(flatten)] - other: IndexMap, -} - -#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] -struct Stage0Toolchain { - date: String, - version: String, -} - #[derive(Debug, serde::Serialize, serde::Deserialize)] struct Manifest { date: String,