Use parallel walker instead of rayon

cli/Cargo.toml

@@ -23,11 +23,11 @@ clap = { version = "4.5", features = [
 ] }
 ignore = "0.4"
 once_cell = "1"
-rayon = "1.10"
 serde = { version = "1", features = ["derive"] }
 toml = "0.8"
 typeshare-core = { path = "../core", version = "=1.13.2" }
 log.workspace = true
 flexi_logger.workspace = true
 anyhow = "1"
 clap_complete = "4.5.32"
+crossbeam = "0.8"

cli/src/main.rs

@@ -19,7 +19,7 @@ use clap_complete::aot::generate;
 use flexi_logger::AdaptiveFormat;
 use ignore::{overrides::OverrideBuilder, types::TypesBuilder, WalkBuilder};
 use log::{error, info};
-use rayon::iter::ParallelBridge;
+use parse::parallel_parse;
 #[cfg(feature = "go")]
 use typeshare_core::language::Go;
 use typeshare_core::{
@@ -34,20 +34,18 @@ use typeshare_core::{
 use crate::{
     args::{Args, Command},
     config::Config,
-    parse::{all_types, parse_input, parser_inputs},
+    parse::all_types,
     writer::{write_generated, Output},
 };
 
 fn main() -> anyhow::Result<()> {
     flexi_logger::Logger::try_with_env_or_str("info")?
-        .adaptive_format_for_stderr(AdaptiveFormat::Detailed)
-        .adaptive_format_for_stdout(AdaptiveFormat::Detailed)
+        .adaptive_format_for_stderr(AdaptiveFormat::Opt)
+        .adaptive_format_for_stdout(AdaptiveFormat::Opt)
         .start()?;
 
     let options = Args::parse();
 
-    info!("typeshare started generating types");
-
     if let Some(options) = options.subcommand {
         match options {
             Command::Completions { shell } => {
@@ -65,13 +63,21 @@ fn main() -> anyhow::Result<()> {
     let config_file = options.config_file.as_deref();
 
     if options.output.generate_config {
-        let config = override_configuration(Config::default(), &options)?;
-        config::store_config(&config, config_file).context("Failed to create new config file")?;
-        return Ok(());
+        override_configuration(Config::default(), &options)
+            .and_then(|config| config::store_config(&config, config_file))
+            .inspect_err(|err| error!("typeshare failed to create new config file: {err}"))
+    } else {
+        generate_types(config_file, &options).inspect_err(|err| {
+            error!("typeshare failed to generate types: {err}");
+        })
     }
+}
+
+fn generate_types(config_file: Option<&Path>, options: &Args) -> anyhow::Result<()> {
+    info!("typeshare started generating types");
 
     let config = config::load_config(config_file).context("Unable to read configuration file")?;
-    let config = override_configuration(config, &options)?;
+    let config = override_configuration(config, options)?;
 
     let directories = options.directories.as_slice();
 
@@ -89,36 +95,6 @@ fn main() -> anyhow::Result<()> {
         },
     };
 
-    let mut types = TypesBuilder::new();
-    types
-        .add("rust", "*.rs")
-        .context("Failed to add rust type extensions")?;
-    types.select("rust");
-
-    // This is guaranteed to always have at least one value by the clap configuration
-    let first_root = directories
-        .first()
-        .expect("directories is empty; this shouldn't be possible");
-
-    let overrides = OverrideBuilder::new(first_root)
-        // Don't process files inside of tools/typeshare/
-        .add("!**/tools/typeshare/**")
-        .context("Failed to parse override")?
-        .build()
-        .context("Failed to build override")?;
-
-    let mut walker_builder = WalkBuilder::new(first_root);
-    // Sort walker output for deterministic output across platforms
-    walker_builder
-        .sort_by_file_path(Path::cmp)
-        .types(types.build().context("Failed to build types")?)
-        .overrides(overrides)
-        .follow_links(options.follow_links);
-
-    for root in directories.iter().skip(1) {
-        walker_builder.add(root);
-    }
-
     let destination = if let Some(ref file) = options.output.file {
         Output::File(file)
     } else if let Some(ref folder) = options.output.folder {
@@ -140,43 +116,62 @@ fn main() -> anyhow::Result<()> {
         target_os,
     };
 
-    // The walker ignores directories that are git-ignored. If you need
-    // a git-ignored directory to be processed, add the specific directory to
-    // the list of directories given to typeshare when it's invoked in the
-    // makefiles
-    // TODO: The `ignore` walker supports parallel walking. We should use this
-    // and implement a `ParallelVisitor` that builds up the mapping of parsed
-    // data. That way both walking and parsing are in parallel.
-    // https://docs.rs/ignore/latest/ignore/struct.WalkParallel.html
-    let crate_parsed_data = parse_input(
-        parser_inputs(walker_builder, language_type, multi_file).par_bridge(),
+    let mut parsed_data = parallel_parse(
         &parse_context,
+        walker_builder(directories, options)?,
+        language_type,
     )?;
 
     // Collect all the types into a map of the file name they
     // belong too and the list of type names. Used for generating
     // imports in generated files.
     let import_candidates = if multi_file {
-        all_types(&crate_parsed_data)
+        all_types(&mut parsed_data)
     } else {
         HashMap::new()
     };
 
-    check_parse_errors(&crate_parsed_data)?;
+    check_parse_errors(&parsed_data)?;
 
     info!("typeshare started writing generated types");
 
-    write_generated(
-        destination,
-        lang.as_mut(),
-        crate_parsed_data,
-        import_candidates,
-    )?;
+    write_generated(destination, lang.as_mut(), parsed_data, import_candidates)?;
 
     info!("typeshare finished generating types");
     Ok(())
 }
 
+fn walker_builder(
+    directories: &[std::path::PathBuf],
+    options: &Args,
+) -> anyhow::Result<WalkBuilder> {
+    let mut types = TypesBuilder::new();
+    types
+        .add("rust", "*.rs")
+        .context("Failed to add rust type extensions")?;
+    types.select("rust");
+
+    let first_root = directories
+        .first()
+        .expect("directories is empty; this shouldn't be possible");
+    let overrides = OverrideBuilder::new(first_root)
+        // Don't process files inside of tools/typeshare/
+        .add("!**/tools/typeshare/**")
+        .context("Failed to parse override")?
+        .build()
+        .context("Failed to build override")?;
+    let mut walker_builder = WalkBuilder::new(first_root);
+    walker_builder
+        .sort_by_file_path(Path::cmp)
+        .types(types.build().context("Failed to build types")?)
+        .overrides(overrides)
+        .follow_links(options.follow_links);
+    for root in directories.iter().skip(1) {
+        walker_builder.add(root);
+    }
+    Ok(walker_builder)
+}
+
 /// Get the language trait impl for the given supported language and configuration.
 fn language(
     language_type: SupportedLanguage,