Rollup merge of #111955 - jyn514:step-refactors, r=clubby789

bootstrap: Various Step refactors

This ended up being a bunch of only somewhat related changes, sorry 😓 on the bright side, they're all fairly small  and well-commented in the commit messages.

- Document `ShouldRun::crates`

- Switch Steps from crates to crate_or_deps where possible

    and document why the single remaining place can't switch

- Switch doc::{Std, Rustc} to `crate_or_deps`

    Previously they were using `all_krates` and various hacks to determine
    which crates to document. Switch them to `crate_or_deps` so `ShouldRun`
    tells them which crate to document instead of having to guess.

    This also makes a few other refactors:
    - Remove the now unused `all_krates`; new code should only use
      `crate_or_deps`.
    - Add tests for documenting Std
    - Remove the unnecessary `run_cargo_rustdoc_for` closure so that we only
      run cargo once
    - Give a more helpful error message when documenting a no_std target
    - Use `builder.msg` in the Steps instead of `builder.info`

- Extend `msg` and `description` to work with any subcommand

    Previously `description` only supported `Testing` and `Benchmarking`,
    and `msg` gave weird results for `doc` (it would say `Docing`).

- Add a `make_run_crates` function and use it Rustc and Std

    This fixes the panic from the previous commit.

- Allow checking individual crates

    This is useful for profiling metadata generation.

    This comes very close to removing all_krates, but doesn't quite -
    there's one last usage left in `doc`. This is fixed in a later commit.

- Give a more helpful error when calling `cargo_crates_in_set` for an alias

    Before:
    ```
    thread 'main' panicked at 'no entry found for key', builder.rs:110:30
    ```

    After:
    ```
    thread 'main' panicked at 'missing crate for path library', check.rs:89:26
    ```

src/bootstrap/builder.rs

@@ -103,11 +103,14 @@ impl RunConfig<'_> {
     }
 
     /// Return a list of crate names selected by `run.paths`.
+    #[track_caller]
     pub fn cargo_crates_in_set(&self) -> Interned<Vec<String>> {
         let mut crates = Vec::new();
         for krate in &self.paths {
             let path = krate.assert_single_path();
-            let crate_name = self.builder.crate_paths[&path.path];
+            let Some(crate_name) = self.builder.crate_paths.get(&path.path) else {
+                panic!("missing crate for path {}", path.path.display())
+            };
             crates.push(crate_name.to_string());
         }
         INTERNER.intern_list(crates)
@@ -427,25 +430,6 @@ impl<'a> ShouldRun<'a> {
         }
     }
 
-    /// Indicates it should run if the command-line selects the given crate or
-    /// any of its (local) dependencies.
-    ///
-    /// Compared to `krate`, this treats the dependencies as aliases for the
-    /// same job. Generally it is preferred to use `krate`, and treat each
-    /// individual path separately. For example `./x.py test src/liballoc`
-    /// (which uses `krate`) will test just `liballoc`. However, `./x.py check
-    /// src/liballoc` (which uses `all_krates`) will check all of `libtest`.
-    /// `all_krates` should probably be removed at some point.
-    pub fn all_krates(mut self, name: &str) -> Self {
-        let mut set = BTreeSet::new();
-        for krate in self.builder.in_tree_crates(name, None) {
-            let path = krate.local_path(self.builder);
-            set.insert(TaskPath { path, kind: Some(self.kind) });
-        }
-        self.paths.insert(PathSet::Set(set));
-        self
-    }
-
     /// Indicates it should run if the command-line selects the given crate or
     /// any of its (local) dependencies.
     ///
@@ -458,6 +442,8 @@ impl<'a> ShouldRun<'a> {
     /// Indicates it should run if the command-line selects any of the given crates.
     ///
     /// `make_run` will be called a single time with all matching command-line paths.
+    ///
+    /// Prefer [`ShouldRun::crate_or_deps`] to this function where possible.
     pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self {
         for krate in crates {
             let path = krate.local_path(self.builder);
@@ -487,7 +473,15 @@ impl<'a> ShouldRun<'a> {
         self.paths(&[path])
     }
 
-    // multiple aliases for the same job
+    /// Multiple aliases for the same job.
+    ///
+    /// This differs from [`path`] in that multiple calls to path will end up calling `make_run`
+    /// multiple times, whereas a single call to `paths` will only ever generate a single call to
+    /// `paths`.
+    ///
+    /// This is analogous to `all_krates`, although `all_krates` is gone now. Prefer [`path`] where possible.
+    ///
+    /// [`path`]: ShouldRun::path
     pub fn paths(mut self, paths: &[&str]) -> Self {
         static SUBMODULES_PATHS: OnceCell<Vec<String>> = OnceCell::new();
 
@@ -641,12 +635,16 @@ impl Kind {
         }
     }
 
-    pub fn test_description(&self) -> &'static str {
+    pub fn description(&self) -> String {
         match self {
             Kind::Test => "Testing",
             Kind::Bench => "Benchmarking",
-            _ => panic!("not a test command: {}!", self.as_str()),
+            Kind::Doc => "Documenting",
+            Kind::Run => "Running",
+            Kind::Suggest => "Suggesting",
+            _ => return format!("{self:?}"),
         }
+        .to_owned()
     }
 }
 

src/bootstrap/builder/tests.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::config::{Config, DryRun, TargetSelection};
+use crate::doc::DocumentationFormat;
 use std::thread;
 
 fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config {
@@ -66,6 +67,16 @@ macro_rules! std {
     };
 }
 
+macro_rules! doc_std {
+    ($host:ident => $target:ident, stage = $stage:literal) => {
+        doc::Std::new(
+            $stage,
+            TargetSelection::from_user(stringify!($target)),
+            DocumentationFormat::HTML,
+        )
+    };
+}
+
 macro_rules! rustc {
     ($host:ident => $target:ident, stage = $stage:literal) => {
         compile::Rustc::new(
@@ -144,6 +155,9 @@ fn alias_and_path_for_library() {
         first(cache.all::<compile::Std>()),
         &[std!(A => A, stage = 0), std!(A => A, stage = 1)]
     );
+
+    let mut cache = run_build(&["library".into(), "core".into()], configure("doc", &["A"], &["A"]));
+    assert_eq!(first(cache.all::<doc::Std>()), &[doc_std!(A => A, stage = 0)]);
 }
 
 #[test]

src/bootstrap/check.rs

@@ -1,8 +1,10 @@
 //! Implementation of compiling the compiler and standard library, in "check"-based modes.
 
-use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
+use crate::builder::{crate_description, Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::cache::Interned;
-use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo};
+use crate::compile::{
+    add_to_sysroot, make_run_crates, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo,
+};
 use crate::config::TargetSelection;
 use crate::tool::{prepare_tool_cargo, SourceType};
 use crate::INTERNER;
@@ -12,6 +14,12 @@ use std::path::{Path, PathBuf};
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Std {
     pub target: TargetSelection,
+    /// Whether to build only a subset of crates.
+    ///
+    /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`].
+    ///
+    /// [`compile::Rustc`]: crate::compile::Rustc
+    crates: Interned<Vec<String>>,
 }
 
 /// Returns args for the subcommand itself (not for cargo)
@@ -66,16 +74,23 @@ fn cargo_subcommand(kind: Kind) -> &'static str {
     }
 }
 
+impl Std {
+    pub fn new(target: TargetSelection) -> Self {
+        Self { target, crates: INTERNER.intern_list(vec![]) }
+    }
+}
+
 impl Step for Std {
     type Output = ();
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.all_krates("sysroot").path("library")
+        run.crate_or_deps("sysroot").path("library")
     }
 
     fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(Std { target: run.target });
+        let crates = make_run_crates(&run, "library");
+        run.builder.ensure(Std { target: run.target, crates });
     }
 
     fn run(self, builder: &Builder<'_>) {
@@ -97,7 +112,14 @@ impl Step for Std {
             cargo.arg("--lib");
         }
 
-        let _guard = builder.msg_check("library artifacts", target);
+        for krate in &*self.crates {
+            cargo.arg("-p").arg(krate);
+        }
+
+        let _guard = builder.msg_check(
+            format_args!("library artifacts{}", crate_description(&self.crates)),
+            target,
+        );
         run_cargo(
             builder,
             cargo,
@@ -117,7 +139,8 @@ impl Step for Std {
         }
 
         // don't run on std twice with x.py clippy
-        if builder.kind == Kind::Clippy {
+        // don't check test dependencies if we haven't built libtest
+        if builder.kind == Kind::Clippy || !self.crates.is_empty() {
             return;
         }
 
@@ -147,8 +170,8 @@ impl Step for Std {
         // Explicitly pass -p for all dependencies krates -- this will force cargo
         // to also check the tests/benches/examples for these crates, rather
         // than just the leaf crate.
-        for krate in builder.in_tree_crates("test", Some(target)) {
-            cargo.arg("-p").arg(krate.name);
+        for krate in &*self.crates {
+            cargo.arg("-p").arg(krate);
         }
 
         let _guard = builder.msg_check("library test/bench/example targets", target);
@@ -167,6 +190,22 @@ impl Step for Std {
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Rustc {
     pub target: TargetSelection,
+    /// Whether to build only a subset of crates.
+    ///
+    /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`].
+    ///
+    /// [`compile::Rustc`]: crate::compile::Rustc
+    crates: Interned<Vec<String>>,
+}
+
+impl Rustc {
+    pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self {
+        let mut crates = vec![];
+        for krate in builder.in_tree_crates("rustc-main", None) {
+            crates.push(krate.name.to_string());
+        }
+        Self { target, crates: INTERNER.intern_list(crates) }
+    }
 }
 
 impl Step for Rustc {
@@ -175,11 +214,12 @@ impl Step for Rustc {
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.all_krates("rustc-main").path("compiler")
+        run.crate_or_deps("rustc-main").path("compiler")
     }
 
     fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(Rustc { target: run.target });
+        let crates = make_run_crates(&run, "compiler");
+        run.builder.ensure(Rustc { target: run.target, crates });
     }
 
     /// Builds the compiler.
@@ -200,7 +240,7 @@ impl Step for Rustc {
             builder.ensure(crate::compile::Std::new(compiler, compiler.host));
             builder.ensure(crate::compile::Std::new(compiler, target));
         } else {
-            builder.ensure(Std { target });
+            builder.ensure(Std::new(target));
         }
 
         let mut cargo = builder.cargo(
@@ -218,14 +258,17 @@ impl Step for Rustc {
             cargo.arg("--all-targets");
         }
 
-        // Explicitly pass -p for all compiler krates -- this will force cargo
+        // Explicitly pass -p for all compiler crates -- this will force cargo
         // to also check the tests/benches/examples for these crates, rather
         // than just the leaf crate.
-        for krate in builder.in_tree_crates("rustc-main", Some(target)) {
-            cargo.arg("-p").arg(krate.name);
+        for krate in &*self.crates {
+            cargo.arg("-p").arg(krate);
         }
 
-        let _guard = builder.msg_check("compiler artifacts", target);
+        let _guard = builder.msg_check(
+            format_args!("compiler artifacts{}", crate_description(&self.crates)),
+            target,
+        );
         run_cargo(
             builder,
             cargo,
@@ -268,7 +311,7 @@ impl Step for CodegenBackend {
         let target = self.target;
         let backend = self.backend;
 
-        builder.ensure(Rustc { target });
+        builder.ensure(Rustc::new(target, builder));
 
         let mut cargo = builder.cargo(
             compiler,
@@ -318,7 +361,7 @@ impl Step for RustAnalyzer {
         let compiler = builder.compiler(builder.top_stage, builder.config.build);
         let target = self.target;
 
-        builder.ensure(Std { target });
+        builder.ensure(Std::new(target));
 
         let mut cargo = prepare_tool_cargo(
             builder,
@@ -386,7 +429,7 @@ macro_rules! tool_check_step {
                 let compiler = builder.compiler(builder.top_stage, builder.config.build);
                 let target = self.target;
 
-                builder.ensure(Rustc { target });
+                builder.ensure(Rustc::new(target, builder));
 
                 let mut cargo = prepare_tool_cargo(
                     builder,

src/bootstrap/compile.rs

@@ -48,6 +48,17 @@ impl Std {
     }
 }
 
+/// Given an `alias` selected by the `Step` and the paths passed on the command line,
+/// return a list of the crates that should be built.
+///
+/// Normally, people will pass *just* `library` if they pass it.
+/// But it's possible (although strange) to pass something like `library std core`.
+/// Build all crates anyway, as if they hadn't passed the other args.
+pub(crate) fn make_run_crates(run: &RunConfig<'_>, alias: &str) -> Interned<Vec<String>> {
+    let has_alias = run.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias));
+    if has_alias { Default::default() } else { run.cargo_crates_in_set() }
+}
+
 impl Step for Std {
     type Output = ();
     const DEFAULT: bool = true;
@@ -62,16 +73,10 @@ impl Step for Std {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        // Normally, people will pass *just* library if they pass it.
-        // But it's possible (although strange) to pass something like `library std core`.
-        // Build all crates anyway, as if they hadn't passed the other args.
-        let has_library =
-            run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library"));
-        let crates = if has_library { Default::default() } else { run.cargo_crates_in_set() };
         run.builder.ensure(Std {
             compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
             target: run.target,
-            crates,
+            crates: make_run_crates(&run, "library"),
         });
     }
 
@@ -615,6 +620,8 @@ impl Step for Rustc {
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         let mut crates = run.builder.in_tree_crates("rustc-main", None);
         for (i, krate) in crates.iter().enumerate() {
+            // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`.
+            // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change.
             if krate.name == "rustc-main" {
                 crates.swap_remove(i);
                 break;

src/bootstrap/dist.rs

@@ -106,11 +106,7 @@ impl Step for JsonDocs {
     /// Builds the `rust-docs-json` installer component.
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
         let host = self.host;
-        builder.ensure(crate::doc::Std {
-            stage: builder.top_stage,
-            target: host,
-            format: DocumentationFormat::JSON,
-        });
+        builder.ensure(crate::doc::Std::new(builder.top_stage, host, DocumentationFormat::JSON));
 
         let dest = "share/doc/rust/json";