Auto merge of #124883 - onur-ozkan:change-stage0-file, r=Mark-Simulacrum

use key-value format in stage0 file

Currently, we are working on the python removal task on bootstrap. Which means we have to extract some data from the stage0 file using shell scripts. However, parsing values from the stage0.json file is painful because shell scripts don't have a built-in way to parse json files.

This change simplifies the stage0 file format to key-value pairs, which makes it easily readable from any environment.

See the zulip thread for more details: https://rust-lang.zulipchat.com/#narrow/stream/326414-t-infra.2Fbootstrap/topic/Using.20different.20format.20in.20the.20stage0.20file

.github/workflows/dependencies.yml

@@ -56,7 +56,7 @@ jobs:
       - name: install the bootstrap toolchain
         run: |
           # Extract the stage0 version
-          TOOLCHAIN=$(jq -r '.compiler | {version,date} | join("-")' -- src/stage0.json)
+          TOOLCHAIN=$(awk -F= '{a[$1]=$2} END {print(a["compiler_version"] "-" a["compiler_date"])}' src/stage0)
           # Install and set as default
           rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN
           rustup default $TOOLCHAIN

Cargo.lock

@@ -381,10 +381,10 @@ name = "bump-stage0"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "build_helper",
  "curl",
  "indexmap",
  "serde",
- "serde_json",
  "toml 0.5.11",
 ]
 

config.example.toml

@@ -213,17 +213,17 @@
 # the root of the repository.
 #build-dir = "build"
 
-# Instead of downloading the src/stage0.json version of Cargo specified, use
+# Instead of downloading the src/stage0 version of Cargo specified, use
 # this Cargo binary instead to build all Rust code
 # If you set this, you likely want to set `rustc` as well.
 #cargo = "/path/to/cargo"
 
-# Instead of downloading the src/stage0.json version of the compiler
+# Instead of downloading the src/stage0 version of the compiler
 # specified, use this rustc binary instead as the stage0 snapshot compiler.
 # If you set this, you likely want to set `cargo` as well.
 #rustc = "/path/to/rustc"
 
-# Instead of downloading the src/stage0.json version of rustfmt specified,
+# Instead of downloading the src/stage0 version of rustfmt specified,
 # use this rustfmt binary instead as the stage0 snapshot rustfmt.
 #rustfmt = "/path/to/rustfmt"
 

src/bootstrap/bootstrap.py

@@ -3,7 +3,6 @@
 import contextlib
 import datetime
 import hashlib
-import json
 import os
 import re
 import shutil
@@ -52,7 +51,7 @@ def get(base, url, path, checksums, verbose=False):
 
     try:
         if url not in checksums:
-            raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
+            raise RuntimeError(("src/stage0 doesn't contain a checksum for {}. "
                                 "Pre-built artifacts might not be available for this "
                                 "target at this time, see https://doc.rust-lang.org/nightly"
                                 "/rustc/platform-support.html for more information.")
@@ -421,9 +420,9 @@ def output(filepath):
 
 
 class Stage0Toolchain:
-    def __init__(self, stage0_payload):
-        self.date = stage0_payload["date"]
-        self.version = stage0_payload["version"]
+    def __init__(self, date, version):
+        self.date = date
+        self.version = version
 
     def channel(self):
         return self.version + "-" + self.date
@@ -439,7 +438,7 @@ def __init__(
         bin_root,
         tarball_path,
         tarball_suffix,
-        checksums_sha256,
+        stage0_data,
         pattern,
         verbose,
     ):
@@ -448,7 +447,7 @@ def __init__(
         self.bin_root = bin_root
         self.tarball_path = tarball_path
         self.tarball_suffix = tarball_suffix
-        self.checksums_sha256 = checksums_sha256
+        self.stage0_data = stage0_data
         self.pattern = pattern
         self.verbose = verbose
 
@@ -458,7 +457,7 @@ def download_component(download_info):
             download_info.base_download_url,
             download_info.download_path,
             download_info.tarball_path,
-            download_info.checksums_sha256,
+            download_info.stage0_data,
             verbose=download_info.verbose,
         )
 
@@ -510,11 +509,12 @@ def __init__(self, config_toml="", args=None):
         build_dir = args.build_dir or self.get_toml('build-dir', 'build') or 'build'
         self.build_dir = os.path.abspath(build_dir)
 
-        with open(os.path.join(self.rust_root, "src", "stage0.json")) as f:
-            data = json.load(f)
-        self.checksums_sha256 = data["checksums_sha256"]
-        self.stage0_compiler = Stage0Toolchain(data["compiler"])
-        self.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"]
+        self.stage0_data = parse_stage0_file(os.path.join(self.rust_root, "src", "stage0"))
+        self.stage0_compiler = Stage0Toolchain(
+            self.stage0_data["compiler_date"],
+            self.stage0_data["compiler_version"]
+        )
+        self.download_url = os.getenv("RUSTUP_DIST_SERVER") or self.stage0_data["dist_server"]
 
         self.build = args.build or self.build_triple()
 
@@ -581,7 +581,7 @@ def download_toolchain(self):
                     bin_root=self.bin_root(),
                     tarball_path=os.path.join(rustc_cache, filename),
                     tarball_suffix=tarball_suffix,
-                    checksums_sha256=self.checksums_sha256,
+                    stage0_data=self.stage0_data,
                     pattern=pattern,
                     verbose=self.verbose,
                 )
@@ -1071,6 +1071,16 @@ def parse_args(args):
 
     return parser.parse_known_args(args)[0]
 
+def parse_stage0_file(path):
+    result = {}
+    with open(path, 'r') as file:
+        for line in file:
+            line = line.strip()
+            if line and not line.startswith('#'):
+                key, value = line.split('=', 1)
+                result[key.strip()] = value.strip()
+    return result
+
 def bootstrap(args):
     """Configure, fetch, build and run the initial bootstrap"""
     rust_root = os.path.abspath(os.path.join(__file__, '../../..'))

src/bootstrap/src/core/config/config.rs

@@ -181,7 +181,7 @@ pub struct Config {
     pub test_compare_mode: bool,
     pub color: Color,
     pub patch_binaries_for_nix: Option<bool>,
-    pub stage0_metadata: Stage0Metadata,
+    pub stage0_metadata: build_helper::stage0_parser::Stage0,
     pub android_ndk: Option<PathBuf>,
     /// Whether to use the `c` feature of the `compiler_builtins` crate.
     pub optimized_compiler_builtins: bool,
@@ -351,34 +351,6 @@ pub struct Config {
     pub paths: Vec<PathBuf>,
 }
 
-#[derive(Default, Deserialize, Clone)]
-pub struct Stage0Metadata {
-    pub compiler: CompilerMetadata,
-    pub config: Stage0Config,
-    pub checksums_sha256: HashMap<String, String>,
-    pub rustfmt: Option<RustfmtMetadata>,
-}
-#[derive(Default, Deserialize, Clone)]
-pub struct CompilerMetadata {
-    pub date: String,
-    pub version: String,
-}
-
-#[derive(Default, Deserialize, Clone)]
-pub struct Stage0Config {
-    pub dist_server: String,
-    pub artifacts_server: String,
-    pub artifacts_with_llvm_assertions_server: String,
-    pub git_merge_commit_email: String,
-    pub git_repository: String,
-    pub nightly_branch: String,
-}
-#[derive(Default, Deserialize, Clone)]
-pub struct RustfmtMetadata {
-    pub date: String,
-    pub version: String,
-}
-
 #[derive(Clone, Debug, Default)]
 pub enum RustfmtState {
     SystemToolchain(PathBuf),
@@ -1298,13 +1270,13 @@ impl Config {
                 Some(p) => PathBuf::from(p),
                 None => git_root,
             };
-            // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when,
+            // If this doesn't have at least `stage0`, we guessed wrong. This can happen when,
             // for example, the build directory is inside of another unrelated git directory.
             // In that case keep the original `CARGO_MANIFEST_DIR` handling.
             //
             // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside
             // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1.
-            if git_root.join("src").join("stage0.json").exists() {
+            if git_root.join("src").join("stage0").exists() {
                 config.src = git_root;
             }
         } else {
@@ -1322,9 +1294,7 @@ impl Config {
             .to_path_buf();
         }
 
-        let stage0_json = t!(std::fs::read(config.src.join("src").join("stage0.json")));
-
-        config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
+        config.stage0_metadata = build_helper::stage0_parser::parse_stage0_file();
 
         // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory.
         let toml_path = flags

src/bootstrap/src/core/download.rs

@@ -9,9 +9,9 @@ use std::{
 };
 
 use build_helper::ci::CiEnv;
+use build_helper::stage0_parser::VersionMetadata;
 use xz2::bufread::XzDecoder;
 
-use crate::core::config::RustfmtMetadata;
 use crate::utils::helpers::{check_run, exe, program_out_of_date};
 use crate::{core::build_steps::llvm::detect_llvm_sha, utils::helpers::hex_encode};
 use crate::{t, Config};
@@ -408,7 +408,7 @@ impl Config {
     /// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't
     /// reuse target directories or artifacts
     pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
-        let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
+        let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
         let channel = format!("{version}-{date}");
 
         let host = self.build;
@@ -606,7 +606,7 @@ impl Config {
             DownloadSource::Dist => {
                 let dist_server = env::var("RUSTUP_DIST_SERVER")
                     .unwrap_or(self.stage0_metadata.config.dist_server.to_string());
-                // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
+                // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0
                 (dist_server, format!("dist/{key}/{filename}"), true)
             }
         };
@@ -616,7 +616,7 @@ impl Config {
         // this on each and every nightly ...
         let checksum = if should_verify {
             let error = format!(
-                "src/stage0.json doesn't contain a checksum for {url}. \
+                "src/stage0 doesn't contain a checksum for {url}. \
                 Pre-built artifacts might not be available for this \
                 target at this time, see https://doc.rust-lang.org/nightly\
                 /rustc/platform-support.html for more information."