From 27b95016d45cd1c0dcaef014373d4bb04d5d30ef Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Fri, 27 May 2022 17:58:32 -0500 Subject: [PATCH 1/5] Rename `download_component` -> `download_ci_component` It was confusing to have two functions with the same name but different behavior. --- src/bootstrap/config.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 6cb0bd518e278..147a4a38c9c34 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1459,12 +1459,12 @@ fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { } let filename = format!("rust-std-{CHANNEL}-{host}.tar.xz"); let pattern = format!("rust-std-{host}"); - download_component(builder, filename, &pattern, commit); + download_ci_component(builder, filename, &pattern, commit); let filename = format!("rustc-{CHANNEL}-{host}.tar.xz"); - download_component(builder, filename, "rustc", commit); + download_ci_component(builder, filename, "rustc", commit); // download-rustc doesn't need its own cargo, it can just use beta's. let filename = format!("rustc-dev-{CHANNEL}-{host}.tar.xz"); - download_component(builder, filename, "rustc-dev", commit); + download_ci_component(builder, filename, "rustc-dev", commit); builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); @@ -1481,7 +1481,7 @@ fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident -fn download_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { +fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { let cache_dst = builder.out.join("cache"); let rustc_cache = cache_dst.join(commit); if !rustc_cache.exists() { From bd6409ddef6ae0be42e91c8c36ddbce984311773 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Fri, 27 May 2022 18:05:32 -0500 Subject: [PATCH 2/5] Simplify handling of `initial_rustfmt` --- src/bootstrap/config.rs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 147a4a38c9c34..2a6b54ce05577 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -859,9 +859,6 @@ impl Config { set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); config.tools = build.tools; - if build.rustfmt.is_some() { - config.initial_rustfmt = build.rustfmt; - } set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); set(&mut config.profiler, build.profiler); @@ -1154,17 +1151,12 @@ impl Config { set(&mut config.missing_tools, t.missing_tools); } - config.initial_rustfmt = config.initial_rustfmt.or_else({ - let build = config.build; - let initial_rustc = &config.initial_rustc; - - move || { - // Cargo does not provide a RUSTFMT environment variable, so we - // synthesize it manually. - let rustfmt = initial_rustc.with_file_name(exe("rustfmt", build)); + config.initial_rustfmt = build.rustfmt.or_else(|| { + // Cargo does not provide a RUSTFMT environment variable, so we + // synthesize it manually. + let rustfmt = config.initial_rustc.with_file_name(exe("rustfmt", config.build)); - if rustfmt.exists() { Some(rustfmt) } else { None } - } + if rustfmt.exists() { Some(rustfmt) } else { None } }); // Now that we've reached the end of our configuration, infer the From 81f511cc2b70a539357f39ba0dfd223224fe5d88 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 28 May 2022 21:17:28 -0500 Subject: [PATCH 3/5] Move beta rustfmt downloads to rustbuild --- src/bootstrap/bootstrap.py | 84 ++++++----------------- src/bootstrap/builder.rs | 7 +- src/bootstrap/config.rs | 133 ++++++++++++++++++++++++++++++++----- src/bootstrap/format.rs | 18 ++--- src/bootstrap/lib.rs | 2 +- src/bootstrap/test.rs | 4 +- 6 files changed, 153 insertions(+), 95 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index d81874bfe7e97..635e4f3703b1c 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -63,31 +63,30 @@ def support_xz(): except tarfile.CompressionError: return False -def get(base, url, path, checksums, verbose=False, do_verify=True): +def get(base, url, path, checksums, verbose=False): with tempfile.NamedTemporaryFile(delete=False) as temp_file: temp_path = temp_file.name try: - if do_verify: - if url not in checksums: - raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. " - "Pre-built artifacts might not available for this " - "target at this time, see https://doc.rust-lang.org/nightly" - "/rustc/platform-support.html for more information.") - .format(url)) - sha256 = checksums[url] - if os.path.exists(path): - if verify(path, sha256, False): - if verbose: - print("using already-download file", path) - return - else: - if verbose: - print("ignoring already-download file", - path, "due to failed verification") - os.unlink(path) + if url not in checksums: + raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. " + "Pre-built artifacts might not be available for this " + "target at this time, see https://doc.rust-lang.org/nightly" + "/rustc/platform-support.html for more information.") + .format(url)) + sha256 = checksums[url] + if os.path.exists(path): + if verify(path, sha256, False): + if verbose: + print("using already-download file", path) + return + else: + if verbose: + print("ignoring already-download file", + path, "due to failed verification") + os.unlink(path) download(temp_path, "{}/{}".format(base, url), True, verbose) - if do_verify and not verify(temp_path, sha256, verbose): + if not verify(temp_path, sha256, verbose): raise RuntimeError("failed verification") if verbose: print("moving {} to {}".format(temp_path, path)) @@ -430,7 +429,6 @@ class RustBuild(object): def __init__(self): self.checksums_sha256 = {} self.stage0_compiler = None - self.stage0_rustfmt = None self._download_url = '' self.build = '' self.build_dir = '' @@ -484,31 +482,10 @@ def download_toolchain(self): with output(self.rustc_stamp()) as rust_stamp: rust_stamp.write(key) - if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( - not os.path.exists(self.rustfmt()) - or self.program_out_of_date( - self.rustfmt_stamp(), - "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel() - ) - ): - if self.stage0_rustfmt is not None: - tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' - filename = "rustfmt-{}-{}{}".format( - self.stage0_rustfmt.version, self.build, tarball_suffix, - ) - self._download_component_helper( - filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date - ) - self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root)) - self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root)) - with output(self.rustfmt_stamp()) as rustfmt_stamp: - rustfmt_stamp.write(self.stage0_rustfmt.channel()) - def _download_component_helper( - self, filename, pattern, tarball_suffix, key=None + self, filename, pattern, tarball_suffix, ): - if key is None: - key = self.stage0_compiler.date + key = self.stage0_compiler.date cache_dst = os.path.join(self.build_dir, "cache") rustc_cache = os.path.join(cache_dst, key) if not os.path.exists(rustc_cache): @@ -524,7 +501,6 @@ def _download_component_helper( tarball, self.checksums_sha256, verbose=self.verbose, - do_verify=True, ) unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) @@ -634,16 +610,6 @@ def rustc_stamp(self): """ return os.path.join(self.bin_root(), '.rustc-stamp') - def rustfmt_stamp(self): - """Return the path for .rustfmt-stamp - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") - True - """ - return os.path.join(self.bin_root(), '.rustfmt-stamp') - def program_out_of_date(self, stamp_path, key): """Check if the given program stamp is out of date""" if not os.path.exists(stamp_path) or self.clean: @@ -717,12 +683,6 @@ def rustc(self): """Return config path for rustc""" return self.program_config('rustc') - def rustfmt(self): - """Return config path for rustfmt""" - if self.stage0_rustfmt is None: - return None - return self.program_config('rustfmt') - def program_config(self, program): """Return config path for the given program at the given stage @@ -1082,8 +1042,6 @@ def bootstrap(help_triggered): data = json.load(f) build.checksums_sha256 = data["checksums_sha256"] build.stage0_compiler = Stage0Toolchain(data["compiler"]) - if data.get("rustfmt") is not None: - build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"]) build.set_dist_environment(data["dist_server"]) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index da13374cee7cd..47551c5082e4b 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -728,7 +728,8 @@ impl<'a> Builder<'a> { Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), Subcommand::Run { ref paths } => (Kind::Run, &paths[..]), - Subcommand::Format { .. } | Subcommand::Clean { .. } | Subcommand::Setup { .. } => { + Subcommand::Format { .. } => (Kind::Format, &[][..]), + Subcommand::Clean { .. } | Subcommand::Setup { .. } => { panic!() } }; @@ -1192,6 +1193,10 @@ impl<'a> Builder<'a> { Config::download_rustc(self) } + pub(crate) fn initial_rustfmt(&self) -> Option { + Config::initial_rustfmt(self) + } + /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 2a6b54ce05577..1beb198340711 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -3,7 +3,7 @@ //! This module implements parsing `config.toml` configuration files to tweak //! how the build runs. -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; @@ -204,10 +204,27 @@ pub struct Config { // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, - pub initial_rustfmt: Option, + #[cfg(not(test))] + initial_rustfmt: RefCell, + #[cfg(test)] + pub initial_rustfmt: RefCell, pub out: PathBuf, } +#[derive(Clone, Debug)] +pub enum RustfmtState { + SystemToolchain(PathBuf), + Downloaded(PathBuf), + Unavailable, + LazyEvaluated, +} + +impl Default for RustfmtState { + fn default() -> Self { + RustfmtState::LazyEvaluated + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub enum LlvmLibunwind { No, @@ -1151,13 +1168,22 @@ impl Config { set(&mut config.missing_tools, t.missing_tools); } - config.initial_rustfmt = build.rustfmt.or_else(|| { - // Cargo does not provide a RUSTFMT environment variable, so we - // synthesize it manually. - let rustfmt = config.initial_rustc.with_file_name(exe("rustfmt", config.build)); - - if rustfmt.exists() { Some(rustfmt) } else { None } - }); + if let Some(r) = build.rustfmt { + *config.initial_rustfmt.borrow_mut() = if r.exists() { + RustfmtState::SystemToolchain(r) + } else { + RustfmtState::Unavailable + }; + } else { + // If using a system toolchain for bootstrapping, see if that has rustfmt available. + let host = config.build; + let rustfmt_path = config.initial_rustc.with_file_name(exe("rustfmt", host)); + let bin_root = config.out.join(host.triple).join("stage0"); + if !rustfmt_path.starts_with(&bin_root) { + // Using a system-provided toolchain; we shouldn't download rustfmt. + *config.initial_rustfmt.borrow_mut() = RustfmtState::SystemToolchain(rustfmt_path); + } + } // Now that we've reached the end of our configuration, infer the // default values for all options that we haven't otherwise stored yet. @@ -1327,6 +1353,25 @@ impl Config { }) } + pub(crate) fn initial_rustfmt(builder: &Builder<'_>) -> Option { + match &mut *builder.config.initial_rustfmt.borrow_mut() { + RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), + RustfmtState::Unavailable => None, + r @ RustfmtState::LazyEvaluated => { + if builder.config.dry_run { + return Some(PathBuf::new()); + } + let path = maybe_download_rustfmt(builder); + *r = if let Some(p) = &path { + RustfmtState::Downloaded(p.clone()) + } else { + RustfmtState::Unavailable + }; + path + } + } + } + pub fn verbose(&self) -> bool { self.verbose > 0 } @@ -1437,6 +1482,44 @@ fn download_ci_rustc_commit(download_rustc: Option, verbose: bool) Some(commit.to_string()) } +fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { + #[derive(Deserialize)] + struct Stage0Metadata { + dist_server: String, + rustfmt: Option, + } + #[derive(Deserialize)] + struct RustfmtMetadata { + date: String, + version: String, + } + + let stage0_json = builder.read(&builder.src.join("src").join("stage0.json")); + let metadata = t!(serde_json::from_str::(&stage0_json)); + let RustfmtMetadata { date, version } = metadata.rustfmt?; + let channel = format!("{version}-{date}"); + let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(metadata.dist_server); + dist_server.push_str("/dist"); + + let host = builder.config.build; + let rustfmt_path = builder.config.initial_rustc.with_file_name(exe("rustfmt", host)); + let bin_root = builder.config.out.join(host.triple).join("stage0"); + let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); + if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { + return Some(rustfmt_path); + } + + let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); + download_component(builder, &dist_server, filename, "rustfmt-preview", &date, "stage0"); + assert!(rustfmt_path.exists()); + + builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); + builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); + + builder.create(&rustfmt_stamp, &channel); + Some(rustfmt_path) +} + fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); // FIXME: support downloading artifacts from the beta channel @@ -1474,18 +1557,34 @@ fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { + download_component( + builder, + "https://ci-artifacts.rust-lang.org/rustc-builds", + filename, + prefix, + commit, + "ci-rustc", + ) +} + +fn download_component( + builder: &Builder<'_>, + base_url: &str, + filename: String, + prefix: &str, + key: &str, + destination: &str, +) { let cache_dst = builder.out.join("cache"); - let rustc_cache = cache_dst.join(commit); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); } - let base = "https://ci-artifacts.rust-lang.org"; - let url = format!("rustc-builds/{commit}"); - let tarball = rustc_cache.join(&filename); + let tarball = cache_dir.join(&filename); if !tarball.exists() { - builder.download_component(base, &format!("{url}/{filename}"), &tarball, ""); + builder.download_component(base_url, &format!("{key}/{filename}"), &tarball, ""); } - let bin_root = builder.out.join(builder.config.build.triple).join("ci-rustc"); + let bin_root = builder.out.join(builder.config.build.triple).join(destination); builder.unpack(&tarball, &bin_root, prefix) } diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index d1a450f1bff8e..60a53c28686b0 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -1,7 +1,7 @@ //! Runs rustfmt on the repository. +use crate::builder::Builder; use crate::util::{output, t}; -use crate::Build; use ignore::WalkBuilder; use std::collections::VecDeque; use std::path::{Path, PathBuf}; @@ -42,7 +42,7 @@ struct RustfmtConfig { ignore: Vec, } -pub fn format(build: &Build, check: bool, paths: &[PathBuf]) { +pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { if build.config.dry_run { return; } @@ -112,15 +112,11 @@ pub fn format(build: &Build, check: bool, paths: &[PathBuf]) { } let ignore_fmt = ignore_fmt.build().unwrap(); - let rustfmt_path = build - .config - .initial_rustfmt - .as_ref() - .unwrap_or_else(|| { - eprintln!("./x.py fmt is not supported on this channel"); - std::process::exit(1); - }) - .to_path_buf(); + let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { + eprintln!("./x.py fmt is not supported on this channel"); + std::process::exit(1); + }); + assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); let src = build.src.clone(); let (tx, rx): (SyncSender, _) = std::sync::mpsc::sync_channel(128); let walker = match paths.get(0) { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index fab6168bf38f6..4974a1c5b7b45 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -661,7 +661,7 @@ impl Build { self.maybe_update_submodules(); if let Subcommand::Format { check, paths } = &self.config.cmd { - return format::format(self, *check, &paths); + return format::format(&builder::Builder::new(&self), *check, &paths); } if let Subcommand::Clean { all } = self.config.cmd { diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 8a236ec5130b9..fdce078bbedf5 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1010,7 +1010,7 @@ impl Step for Tidy { if builder.config.channel == "dev" || builder.config.channel == "nightly" { builder.info("fmt check"); - if builder.config.initial_rustfmt.is_none() { + if builder.initial_rustfmt().is_none() { let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap(); eprintln!( "\ @@ -1023,7 +1023,7 @@ help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` ); std::process::exit(1); } - crate::format::format(&builder.build, !builder.config.cmd.bless(), &[]); + crate::format::format(&builder, !builder.config.cmd.bless(), &[]); } } From a9ca4b95295ce84ec1ba89a657647e2de03bb132 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 28 May 2022 22:05:43 -0500 Subject: [PATCH 4/5] Add checksum verification for rustfmt downloads --- Cargo.lock | 2 ++ src/bootstrap/Cargo.toml | 2 ++ src/bootstrap/builder.rs | 23 +++++++++++++- src/bootstrap/config.rs | 66 ++++++++++++++++++++++++++++++++++------ 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a9b35fe4aba9..b6077574ee600 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,6 +216,7 @@ dependencies = [ "cmake", "filetime", "getopts", + "hex 0.4.2", "ignore", "libc", "once_cell", @@ -223,6 +224,7 @@ dependencies = [ "pretty_assertions 0.7.2", "serde", "serde_json", + "sha2", "sysinfo", "tar", "toml", diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 5027a45e0ada0..0e54837610a4b 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -40,8 +40,10 @@ filetime = "0.2" getopts = "0.2.19" cc = "1.0.69" libc = "0.2" +hex = "0.4" serde = { version = "1.0.8", features = ["derive"] } serde_json = "1.0.2" +sha2 = "0.10" tar = "0.4" toml = "0.5" ignore = "0.4.10" diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 47551c5082e4b..38d4f15d3c858 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -879,7 +879,6 @@ impl<'a> Builder<'a> { ) { // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - // FIXME: support `do_verify` (only really needed for nightly rustfmt) self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error); t!(std::fs::rename(&tempfile, dest_path)); } @@ -971,6 +970,28 @@ impl<'a> Builder<'a> { t!(fs::remove_dir_all(dst.join(directory_prefix))); } + /// Returns whether the SHA256 checksum of `path` matches `expected`. + pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { + use sha2::Digest; + + self.verbose(&format!("verifying {}", path.display())); + let mut hasher = sha2::Sha256::new(); + // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. + // Consider using streaming IO instead? + let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) }; + hasher.update(&contents); + let found = hex::encode(hasher.finalize().as_slice()); + let verified = found == expected; + if !verified && !self.config.dry_run { + println!( + "invalid checksum: \n\ + found: {found}\n\ + expected: {expected}", + ); + } + return verified; + } + /// Obtain a compiler at a given stage and for a given host. Explicitly does /// not take `Compiler` since all `Compiler` instances are meant to be /// obtained through this function, since it ensures that they are valid diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 1beb198340711..70bd7473c872d 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1486,6 +1486,7 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { #[derive(Deserialize)] struct Stage0Metadata { dist_server: String, + checksums_sha256: HashMap, rustfmt: Option, } #[derive(Deserialize)] @@ -1495,10 +1496,11 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { } let stage0_json = builder.read(&builder.src.join("src").join("stage0.json")); - let metadata = t!(serde_json::from_str::(&stage0_json)); - let RustfmtMetadata { date, version } = metadata.rustfmt?; + let Stage0Metadata { dist_server, checksums_sha256, rustfmt } = + t!(serde_json::from_str::(&stage0_json)); + let RustfmtMetadata { date, version } = rustfmt?; let channel = format!("{version}-{date}"); - let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(metadata.dist_server); + let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(dist_server); dist_server.push_str("/dist"); let host = builder.config.build; @@ -1510,8 +1512,15 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { } let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); - download_component(builder, &dist_server, filename, "rustfmt-preview", &date, "stage0"); - assert!(rustfmt_path.exists()); + download_component( + builder, + &dist_server, + filename, + "rustfmt-preview", + &date, + "stage0", + Some(checksums_sha256), + ); builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); @@ -1564,6 +1573,7 @@ fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, prefix, commit, "ci-rustc", + None, ) } @@ -1574,6 +1584,7 @@ fn download_component( prefix: &str, key: &str, destination: &str, + checksums: Option>, ) { let cache_dst = builder.out.join("cache"); let cache_dir = cache_dst.join(key); @@ -1581,10 +1592,47 @@ fn download_component( t!(fs::create_dir_all(&cache_dir)); } + let bin_root = builder.out.join(builder.config.build.triple).join(destination); let tarball = cache_dir.join(&filename); - if !tarball.exists() { - builder.download_component(base_url, &format!("{key}/{filename}"), &tarball, ""); + let url = format!("{key}/{filename}"); + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if let Some(checksums) = &checksums { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + // TODO: add an enum { Commit, Published } so we don't have to hardcode `dist` in two places + let sha256 = checksums.get(&format!("dist/{url}")).expect(&error); + if tarball.exists() { + if builder.verify(&tarball, sha256) { + builder.unpack(&tarball, &bin_root, prefix); + return; + } else { + builder.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + builder.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + return; + } else { + None + }; + + builder.download_component(base_url, &url, &tarball, ""); + if let Some(sha256) = checksum { + if !builder.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } } - let bin_root = builder.out.join(builder.config.build.triple).join(destination); - builder.unpack(&tarball, &bin_root, prefix) + + builder.unpack(&tarball, &bin_root, prefix); } From 6115f4eba458e763370119b2a10a73ef595583bc Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 28 May 2022 23:55:42 -0500 Subject: [PATCH 5/5] Add a `DownloadSource` enum This simplifies the arguments to `download_component` in config.rs. It also moves stage0.json metadata handling to `Build::new`, making it easier to download the stage0 compiler in rustbuild later if necessary. --- src/bootstrap/config.rs | 69 ++++++++++++++++------------------------- src/bootstrap/lib.rs | 18 +++++++++++ 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 70bd7473c872d..99b69ee9a4fd1 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -20,6 +20,7 @@ use crate::channel::GitInfo; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags}; use crate::util::{exe, output, program_out_of_date, t}; +use crate::RustfmtMetadata; use once_cell::sync::OnceCell; use serde::{Deserialize, Deserializer}; @@ -1483,25 +1484,8 @@ fn download_ci_rustc_commit(download_rustc: Option, verbose: bool) } fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { - #[derive(Deserialize)] - struct Stage0Metadata { - dist_server: String, - checksums_sha256: HashMap, - rustfmt: Option, - } - #[derive(Deserialize)] - struct RustfmtMetadata { - date: String, - version: String, - } - - let stage0_json = builder.read(&builder.src.join("src").join("stage0.json")); - let Stage0Metadata { dist_server, checksums_sha256, rustfmt } = - t!(serde_json::from_str::(&stage0_json)); - let RustfmtMetadata { date, version } = rustfmt?; + let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?; let channel = format!("{version}-{date}"); - let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(dist_server); - dist_server.push_str("/dist"); let host = builder.config.build; let rustfmt_path = builder.config.initial_rustc.with_file_name(exe("rustfmt", host)); @@ -1512,15 +1496,7 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { } let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); - download_component( - builder, - &dist_server, - filename, - "rustfmt-preview", - &date, - "stage0", - Some(checksums_sha256), - ); + download_component(builder, DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0"); builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); @@ -1563,28 +1539,24 @@ fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { } } +pub(crate) enum DownloadSource { + CI, + Dist, +} + /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { - download_component( - builder, - "https://ci-artifacts.rust-lang.org/rustc-builds", - filename, - prefix, - commit, - "ci-rustc", - None, - ) + download_component(builder, DownloadSource::CI, filename, prefix, commit, "ci-rustc") } fn download_component( builder: &Builder<'_>, - base_url: &str, + mode: DownloadSource, filename: String, prefix: &str, key: &str, destination: &str, - checksums: Option>, ) { let cache_dst = builder.out.join("cache"); let cache_dir = cache_dst.join(key); @@ -1594,20 +1566,31 @@ fn download_component( let bin_root = builder.out.join(builder.config.build.triple).join(destination); let tarball = cache_dir.join(&filename); - let url = format!("{key}/{filename}"); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => ( + "https://ci-artifacts.rust-lang.org/rustc-builds".to_string(), + format!("{key}/{filename}"), + false, + ), + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(builder.stage0_metadata.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; // For the beta compiler, put special effort into ensuring the checksums are valid. // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update // this on each and every nightly ... - let checksum = if let Some(checksums) = &checksums { + let checksum = if should_verify { let error = format!( "src/stage0.json doesn't contain a checksum for {url}. \ Pre-built artifacts might not be available for this \ target at this time, see https://doc.rust-lang.org/nightly\ /rustc/platform-support.html for more information." ); - // TODO: add an enum { Commit, Published } so we don't have to hardcode `dist` in two places - let sha256 = checksums.get(&format!("dist/{url}")).expect(&error); + let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error); if tarball.exists() { if builder.verify(&tarball, sha256) { builder.unpack(&tarball, &bin_root, prefix); @@ -1627,7 +1610,7 @@ fn download_component( None }; - builder.download_component(base_url, &url, &tarball, ""); + builder.download_component(&base_url, &url, &tarball, ""); if let Some(sha256) = checksum { if !builder.verify(&tarball, sha256) { panic!("failed to verify {}", tarball.display()); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 4974a1c5b7b45..022f2e0fc1387 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -118,6 +118,7 @@ use std::os::windows::fs::symlink_file; use filetime::FileTime; use once_cell::sync::OnceCell; +use serde::Deserialize; use crate::builder::Kind; use crate::config::{LlvmLibunwind, TargetSelection}; @@ -294,6 +295,7 @@ pub struct Build { targets: Vec, // Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents + stage0_metadata: Stage0Metadata, initial_rustc: PathBuf, initial_cargo: PathBuf, initial_lld: PathBuf, @@ -320,6 +322,18 @@ pub struct Build { metrics: metrics::BuildMetrics, } +#[derive(Deserialize)] +struct Stage0Metadata { + dist_server: String, + checksums_sha256: HashMap, + rustfmt: Option, +} +#[derive(Deserialize)] +struct RustfmtMetadata { + date: String, + version: String, +} + #[derive(Debug)] struct Crate { name: Interned, @@ -468,7 +482,11 @@ impl Build { bootstrap_out }; + let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json"))); + let stage0_metadata = t!(serde_json::from_str::(&stage0_json)); + let mut build = Build { + stage0_metadata, initial_rustc: config.initial_rustc.clone(), initial_cargo: config.initial_cargo.clone(), initial_lld,