diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d33d5..12c5676 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog for Ferium +## `v3.28.0` +### 08.05.2022 + +Upgrading and verbose listing of mods is now _**SUPER**_ fast compared to before (14-20 times) due to multi threading + +- Added multi threading for getting latest mod versions and downloading mods +- Added `--threads` options to limit the maximum number of additional threads +- Used `Arc` in many locations to use the APIs without having to _actually_ clone them +- Added `mutex_ext` to (somewhat unsafely) recover from a poison error and lock a mutex +- If a CurseForge request fails during version determination with a status code, then the request is tried again + - Requests are sent so fast the CF API gives 500 internal server errors sometimes + ## `v3.27.0` ### 07.05.2022 diff --git a/Cargo.lock b/Cargo.lock index 995030c..82ab9e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -493,7 +493,7 @@ dependencies = [ [[package]] name = "ferium" -version = "3.27.0" +version = "3.28.0" dependencies = [ "anyhow", "bytes", diff --git a/Cargo.toml b/Cargo.toml index f5702aa..be34d72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ferium" -version = "3.27.0" +version = "3.28.0" edition = "2021" authors = ["Ilesh Thiada (theRookieCoder) ", "薛詠謙 (KyleUltimate)", "Daniel Hauck (SolidTux)"] description = "Ferium is a CLI program for managing Minecraft mods from Modrinth, CurseForge, and Github Releases" diff --git a/README.md b/README.md index afe9b16..e58a595 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ Simply specify the mods you use through the CLI and in just one command, you can - Upgrade all your mods in one command, `ferium upgrade` - Ferium checks that the version being downloaded is the latest one compatible with the chosen mod loader and Minecraft version - Create multiple profiles and configure different mod loaders, Minecraft versions, output directories, and mods for each -- Configure overrides for mods that are not specified as compatible, but still work +- Configure overrides for mods that are not specified as compatible but still work +- Multi-threading for network intensive subcommands + - You can configure the maximum number of additional threads using the `--threads` options ## Installation diff --git a/src/cli.rs b/src/cli.rs index ab61ac5..d1b0519 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -8,6 +8,9 @@ use std::path::PathBuf; pub struct Ferium { #[clap(subcommand)] pub subcommand: SubCommands, + #[clap(long, short)] + #[clap(help("The limit for additional threads spawned by the Tokio runtime"))] + pub threads: Option, #[clap(long)] #[clap(help("A GitHub personal access token for increasing the rate limit"))] pub github_token: Option, diff --git a/src/main.rs b/src/main.rs index 51af15c..82f4de3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ mod cli; +mod mutex_ext; mod subcommands; use anyhow::{anyhow, bail, Result}; @@ -9,8 +10,9 @@ use ferinth::Ferinth; use furse::Furse; use lazy_static::lazy_static; use libium::config; +use std::sync::Arc; use subcommands::{add, upgrade}; -use tokio::{fs::create_dir_all, io::AsyncReadExt}; +use tokio::{fs::create_dir_all, io::AsyncReadExt, runtime, spawn}; const CROSS: &str = "×"; lazy_static! { @@ -20,18 +22,23 @@ lazy_static! { dialoguer::theme::ColorfulTheme::default(); } -#[tokio::main] -async fn main() { - if let Err(err) = actual_main().await { +fn main() { + let cli = Ferium::parse(); + let mut builder = runtime::Builder::new_multi_thread(); + builder.enable_all(); + builder.thread_name("ferium-worker"); + if let Some(threads) = cli.threads { + builder.max_blocking_threads(threads); + } + let runtime = builder.build().expect("Could not initialise Tokio runtime"); + if let Err(err) = runtime.block_on(actual_main(cli)) { eprintln!("{}", err.to_string().red().bold()); + runtime.shutdown_background(); std::process::exit(1); } } -async fn actual_main() -> Result<()> { - // This also displays the help page or version automatically - let cli_app = Ferium::parse(); - +async fn actual_main(cli_app: Ferium) -> Result<()> { let github = { let mut builder = octocrab::OctocrabBuilder::new(); if let Some(token) = cli_app.github_token { @@ -39,11 +46,11 @@ async fn actual_main() -> Result<()> { } octocrab::initialise(builder) }?; - let modrinth = Ferinth::new(); - let curseforge = Furse::new(env!( + let modrinth = Arc::new(Ferinth::new()); + let curseforge = Arc::new(Furse::new(env!( "CURSEFORGE_API_KEY", "A CurseForge API key is required to build. If you don't have one, you can bypass this by setting the variable to a blank string, however anything using the CurseForge API will not work." - )); + ))); let mut config_file = config::get_file(cli_app.config_file.unwrap_or_else(config::file_path)).await?; let mut config_file_contents = String::new(); @@ -144,22 +151,28 @@ async fn actual_main() -> Result<()> { }, SubCommands::List { verbose } => { check_empty_profile(profile)?; - for mod_ in &profile.mods { - if verbose { + if verbose { + check_internet().await?; + let mut tasks = Vec::new(); + for mod_ in &profile.mods { use config::structs::ModIdentifier; - check_internet().await?; match &mod_.identifier { - ModIdentifier::CurseForgeProject(project_id) => { - subcommands::list::curseforge(&curseforge, *project_id).await - }, - ModIdentifier::ModrinthProject(project_id) => { - subcommands::list::modrinth(&modrinth, project_id).await - }, - ModIdentifier::GitHubRepository(full_name) => { - subcommands::list::github(&github, full_name).await - }, - }?; - } else { + ModIdentifier::CurseForgeProject(project_id) => tasks.push(spawn( + subcommands::list::curseforge(curseforge.clone(), *project_id), + )), + ModIdentifier::ModrinthProject(project_id) => tasks.push(spawn( + subcommands::list::modrinth(modrinth.clone(), project_id.clone()), + )), + ModIdentifier::GitHubRepository(full_name) => tasks.push(spawn( + subcommands::list::github(github.clone(), full_name.clone()), + )), + }; + } + for handle in tasks { + handle.await??; + } + } else { + for mod_ in &profile.mods { println!("{}", mod_.name); } } @@ -200,7 +213,7 @@ async fn actual_main() -> Result<()> { check_internet().await?; check_empty_profile(profile)?; create_dir_all(&profile.output_dir.join(".old")).await?; - upgrade(&modrinth, &curseforge, &github, profile).await?; + upgrade(modrinth, curseforge, github, profile).await?; }, }; diff --git a/src/mutex_ext.rs b/src/mutex_ext.rs new file mode 100644 index 0000000..3ae70bc --- /dev/null +++ b/src/mutex_ext.rs @@ -0,0 +1,18 @@ +use std::sync::{Mutex, MutexGuard}; + +/// A sketchy way to not deal with mutex poisoning +/// +/// **WARNING**: If the poison had occured during a write, the data may be corrupted. +/// _If_ unsafe code had poisoned the mutex, memory corruption is possible +pub trait MutexExt { + fn force_lock(&self) -> MutexGuard<'_, T>; +} + +impl MutexExt for Mutex { + fn force_lock(&self) -> MutexGuard<'_, T> { + match self.lock() { + Ok(guard) => guard, + Err(error) => error.into_inner(), + } + } +} diff --git a/src/subcommands/list.rs b/src/subcommands/list.rs index ca9bf19..44f7ead 100644 --- a/src/subcommands/list.rs +++ b/src/subcommands/list.rs @@ -3,8 +3,9 @@ use ferinth::Ferinth; use furse::Furse; use itertools::Itertools; use octocrab::Octocrab; +use std::sync::Arc; -pub async fn curseforge(curseforge: &Furse, project_id: i32) -> Result<()> { +pub async fn curseforge(curseforge: Arc, project_id: i32) -> Result<()> { let project = curseforge.get_mod(project_id).await?; let authors = project .authors @@ -41,8 +42,8 @@ pub async fn curseforge(curseforge: &Furse, project_id: i32) -> Result<()> { Ok(()) } -pub async fn modrinth(modrinth: &Ferinth, project_id: &str) -> Result<()> { - let project = modrinth.get_project(project_id).await?; +pub async fn modrinth(modrinth: Arc, project_id: String) -> Result<()> { + let project = modrinth.get_project(&project_id).await?; let team_members = modrinth.list_team_members(&project.team).await?; // Get the usernames of all the developers @@ -83,7 +84,7 @@ pub async fn modrinth(modrinth: &Ferinth, project_id: &str) -> Result<()> { } /// List all the mods in `profile` with some of their metadata -pub async fn github(github: &Octocrab, full_name: &(String, String)) -> Result<()> { +pub async fn github(github: Arc, full_name: (String, String)) -> Result<()> { let repo_handler = github.repos(&full_name.0, &full_name.1); let repo = repo_handler.get().await?; let releases = repo_handler.releases().list().send().await?; diff --git a/src/subcommands/upgrade.rs b/src/subcommands/upgrade.rs index d11b584..3181148 100644 --- a/src/subcommands/upgrade.rs +++ b/src/subcommands/upgrade.rs @@ -1,4 +1,5 @@ -use crate::{CROSS, TICK, YELLOW_TICK}; +use crate::{mutex_ext::MutexExt, CROSS, TICK, YELLOW_TICK}; +use anyhow::Error; use anyhow::{bail, Result}; use colored::Colorize; use ferinth::Ferinth; @@ -8,9 +9,15 @@ use itertools::Itertools; use libium::config; use libium::upgrade; use octocrab::Octocrab; -use std::fs::read_dir; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{ + fs::read_dir, + sync::{Arc, Mutex}, +}; use tokio::fs::copy; +use tokio::spawn; +#[derive(Debug, Clone)] struct Downloadable { filename: String, download_url: String, @@ -54,131 +61,174 @@ impl From for Downloadable { } pub async fn upgrade( - modrinth: &Ferinth, - curseforge: &Furse, - github: &Octocrab, + modrinth: Arc, + curseforge: Arc, + github: Arc, profile: &config::structs::Profile, ) -> Result<()> { - let mut to_download = Vec::new(); - let mut backwards_compat_msg = false; - let mut error = false; + let profile = Arc::new(profile.clone()); + let to_download = Arc::new(Mutex::new(Vec::new())); + let backwards_compat_msg = Arc::new(AtomicBool::new(false)); + let error = Arc::new(AtomicBool::new(false)); + let mut tasks = Vec::new(); println!("{}\n", "Determining the Latest Compatible Versions".bold()); for mod_ in &profile.mods { - use libium::config::structs::ModIdentifier; - let (result, backwards_compat): (Result, bool) = match &mod_.identifier { - ModIdentifier::CurseForgeProject(project_id) => { - let result = upgrade::curseforge( - curseforge, - *project_id, - &profile.game_version, - &profile.mod_loader, - mod_.check_game_version, - mod_.check_mod_loader, - ) - .await; - if matches!(result, Err(upgrade::Error::NoCompatibleFile)) - && profile.mod_loader == config::structs::ModLoader::Quilt - { - ( - upgrade::curseforge( - curseforge, - *project_id, - &profile.game_version, - &config::structs::ModLoader::Fabric, - mod_.check_game_version, - mod_.check_mod_loader, - ) - .await - .map(Into::into), - true, + let backwards_compat_msg = backwards_compat_msg.clone(); + let to_download = to_download.clone(); + let error = error.clone(); + let curseforge = curseforge.clone(); + let modrinth = modrinth.clone(); + let profile = profile.clone(); + let github = github.clone(); + let mod_ = mod_.clone(); + tasks.push(spawn(async move { + use libium::config::structs::ModIdentifier; + let (result, backwards_compat): (Result, bool) = match &mod_.identifier + { + ModIdentifier::CurseForgeProject(project_id) => { + let result = upgrade::curseforge( + &curseforge, + *project_id, + &profile.game_version, + &profile.mod_loader, + mod_.check_game_version, + mod_.check_mod_loader, ) - } else { - (result.map(Into::into), false) - } - }, - ModIdentifier::ModrinthProject(project_id) => { - let result = upgrade::modrinth( - modrinth, - project_id, - &profile.game_version, - &profile.mod_loader, - mod_.check_game_version, - mod_.check_mod_loader, - ) - .await; - if matches!(result, Err(upgrade::Error::NoCompatibleFile)) - && profile.mod_loader == config::structs::ModLoader::Quilt - { - ( - upgrade::modrinth( - modrinth, - project_id, - &profile.game_version, - &config::structs::ModLoader::Fabric, - mod_.check_game_version, - mod_.check_mod_loader, + .await; + if matches!(result, Err(upgrade::Error::NoCompatibleFile)) + && profile.mod_loader == config::structs::ModLoader::Quilt + { + ( + upgrade::curseforge( + &curseforge, + *project_id, + &profile.game_version, + &config::structs::ModLoader::Fabric, + mod_.check_game_version, + mod_.check_mod_loader, + ) + .await + .map(Into::into), + true, ) - .await - .map(Into::into), - true, + } else if let Err(upgrade::Error::CurseForgeError( + furse::Error::ReqwestError(err), + )) = &result + { + if err.is_status() { + ( + upgrade::curseforge( + &curseforge, + *project_id, + &profile.game_version, + &profile.mod_loader, + mod_.check_game_version, + mod_.check_mod_loader, + ) + .await + .map(Into::into), + false, + ) + } else { + (result.map(Into::into), false) + } + } else { + (result.map(Into::into), false) + } + }, + ModIdentifier::ModrinthProject(project_id) => { + let result = upgrade::modrinth( + &modrinth, + project_id, + &profile.game_version, + &profile.mod_loader, + mod_.check_game_version, + mod_.check_mod_loader, ) - } else { - (result.map(Into::into), false) - } - }, - ModIdentifier::GitHubRepository(full_name) => { - let result = upgrade::github( - &github.repos(&full_name.0, &full_name.1), - &profile.game_version, - &profile.mod_loader, - mod_.check_game_version, - mod_.check_mod_loader, - ) - .await; - if matches!(result, Err(upgrade::Error::NoCompatibleFile)) - && profile.mod_loader == config::structs::ModLoader::Quilt - { - ( - upgrade::github( - &github.repos(&full_name.0, &full_name.1), - &profile.game_version, - &config::structs::ModLoader::Fabric, - mod_.check_game_version, - mod_.check_mod_loader, + .await; + if matches!(result, Err(upgrade::Error::NoCompatibleFile)) + && profile.mod_loader == config::structs::ModLoader::Quilt + { + ( + upgrade::modrinth( + &modrinth, + project_id, + &profile.game_version, + &config::structs::ModLoader::Fabric, + mod_.check_game_version, + mod_.check_mod_loader, + ) + .await + .map(Into::into), + true, ) - .await - .map(Into::into), - true, + } else { + (result.map(Into::into), false) + } + }, + ModIdentifier::GitHubRepository(full_name) => { + let result = upgrade::github( + &github.repos(&full_name.0, &full_name.1), + &profile.game_version, + &profile.mod_loader, + mod_.check_game_version, + mod_.check_mod_loader, ) - } else { - (result.map(Into::into), false) - } - }, - }; - - match result { - Ok(result) => { - println!( - "{} {:40}{}", - if backwards_compat { - backwards_compat_msg = true; - YELLOW_TICK.clone() + .await; + if matches!(result, Err(upgrade::Error::NoCompatibleFile)) + && profile.mod_loader == config::structs::ModLoader::Quilt + { + ( + upgrade::github( + &github.repos(&full_name.0, &full_name.1), + &profile.game_version, + &config::structs::ModLoader::Fabric, + mod_.check_game_version, + mod_.check_mod_loader, + ) + .await + .map(Into::into), + true, + ) } else { - TICK.clone() - }, - mod_.name, - format!("({})", result.filename).dimmed() - ); - to_download.push(result); - }, - Err(err) => { - eprintln!("{}", format!("{} {:40}{}", CROSS, mod_.name, err).red()); - error = true; - }, - } + (result.map(Into::into), false) + } + }, + }; + + match result { + Ok(result) => { + println!( + "{} {:40}{}", + if backwards_compat { + backwards_compat_msg.store(true, Ordering::Relaxed); + YELLOW_TICK.clone() + } else { + TICK.clone() + }, + mod_.name, + format!("({})", result.filename).dimmed() + ); + { + let mut to_download = to_download.force_lock(); + to_download.push(result); + } + }, + Err(err) => { + eprintln!("{}", format!("{} {:40}{}", CROSS, mod_.name, err).red()); + error.store(true, Ordering::Relaxed); + }, + } + })); } - if backwards_compat_msg { + for handle in tasks { + handle.await?; + } + let mut to_download = Arc::try_unwrap(to_download) + .expect("Failed to run threads to completion") + .into_inner()?; + if backwards_compat_msg.load(Ordering::Relaxed) { println!( "{}", "Fabric mod using Quilt backwards compatibility".yellow() @@ -222,14 +272,22 @@ pub async fn upgrade( } } + let mut tasks = Vec::new(); for downloadable in to_download { - eprint!("Downloading {}... ", downloadable.filename.dimmed()); - let contents = reqwest::get(downloadable.download_url) - .await? - .bytes() - .await?; - upgrade::write_mod_file(profile, contents, &downloadable.filename).await?; - println!("{}", &*TICK); + let profile = profile.clone(); + let downloadable = downloadable.clone(); + tasks.push(spawn(async move { + let contents = reqwest::get(&downloadable.download_url) + .await? + .bytes() + .await?; + upgrade::write_mod_file(&profile, contents, &downloadable.filename).await?; + println!("{} Downloaded {}", &*TICK, downloadable.filename.dimmed()); + Ok::<(), Error>(()) + })); + } + for handle in tasks { + handle.await??; } for installable in to_install { eprint!( @@ -240,7 +298,7 @@ pub async fn upgrade( println!("{}", &*TICK); } - if error { + if error.load(Ordering::Relaxed) { bail!("\nCould not get the latest compatible version of some mods") }