Parallelised network requests

Author: theRookieCoder

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog for Ferium
 
+## `v3.28.0`
+### 08.05.2022
+
+Upgrading and verbose listing of mods is now _**SUPER**_ fast compared to before (14-20 times) due to multi threading
+
+- Added multi threading for getting latest mod versions and downloading mods
+- Added `--threads` options to limit the maximum number of additional threads
+- Used `Arc` in many locations to use the APIs without having to _actually_ clone them
+- Added `mutex_ext` to (somewhat unsafely) recover from a poison error and lock a mutex
+- If a CurseForge request fails during version determination with a status code, then the request is tried again
+  - Requests are sent so fast the CF API gives 500 internal server errors sometimes
+
 ## `v3.27.0`
 ### 07.05.2022
 

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "ferium"
-version = "3.27.0"
+version = "3.28.0"
 edition = "2021"
 authors = ["Ilesh Thiada (theRookieCoder) <[email protected]>", "薛詠謙 (KyleUltimate)", "Daniel Hauck (SolidTux)"]
 description = "Ferium is a CLI program for managing Minecraft mods from Modrinth, CurseForge, and Github Releases"

Cargo.toml

@@ -15,7 +15,9 @@ Simply specify the mods you use through the CLI and in just one command, you can
 - Upgrade all your mods in one command, `ferium upgrade`
   - Ferium checks that the version being downloaded is the latest one compatible with the chosen mod loader and Minecraft version
 - Create multiple profiles and configure different mod loaders, Minecraft versions, output directories, and mods for each
-- Configure overrides for mods that are not specified as compatible, but still work
+- Configure overrides for mods that are not specified as compatible but still work
+- Multi-threading for network intensive subcommands
+  - You can configure the maximum number of additional threads using the `--threads` options
 
 ## Installation
 

README.md

@@ -8,6 +8,9 @@ use std::path::PathBuf;
 pub struct Ferium {
     #[clap(subcommand)]
     pub subcommand: SubCommands,
+    #[clap(long, short)]
+    #[clap(help("The limit for additional threads spawned by the Tokio runtime"))]
+    pub threads: Option<usize>,
     #[clap(long)]
     #[clap(help("A GitHub personal access token for increasing the rate limit"))]
     pub github_token: Option<String>,

src/cli.rs

@@ -1,4 +1,5 @@
 mod cli;
+mod mutex_ext;
 mod subcommands;
 
 use anyhow::{anyhow, bail, Result};
@@ -9,8 +10,9 @@ use ferinth::Ferinth;
 use furse::Furse;
 use lazy_static::lazy_static;
 use libium::config;
+use std::sync::Arc;
 use subcommands::{add, upgrade};
-use tokio::{fs::create_dir_all, io::AsyncReadExt};
+use tokio::{fs::create_dir_all, io::AsyncReadExt, runtime, spawn};
 
 const CROSS: &str = "×";
 lazy_static! {
@@ -20,30 +22,35 @@ lazy_static! {
         dialoguer::theme::ColorfulTheme::default();
 }
 
-#[tokio::main]
-async fn main() {
-    if let Err(err) = actual_main().await {
+fn main() {
+    let cli = Ferium::parse();
+    let mut builder = runtime::Builder::new_multi_thread();
+    builder.enable_all();
+    builder.thread_name("ferium-worker");
+    if let Some(threads) = cli.threads {
+        builder.max_blocking_threads(threads);
+    }
+    let runtime = builder.build().expect("Could not initialise Tokio runtime");
+    if let Err(err) = runtime.block_on(actual_main(cli)) {
         eprintln!("{}", err.to_string().red().bold());
+        runtime.shutdown_background();
         std::process::exit(1);
     }
 }
 
-async fn actual_main() -> Result<()> {
-    // This also displays the help page or version automatically
-    let cli_app = Ferium::parse();
-
+async fn actual_main(cli_app: Ferium) -> Result<()> {
     let github = {
         let mut builder = octocrab::OctocrabBuilder::new();
         if let Some(token) = cli_app.github_token {
             builder = builder.personal_token(token);
         }
         octocrab::initialise(builder)
     }?;
-    let modrinth = Ferinth::new();
-    let curseforge = Furse::new(env!(
+    let modrinth = Arc::new(Ferinth::new());
+    let curseforge = Arc::new(Furse::new(env!(
         "CURSEFORGE_API_KEY",
         "A CurseForge API key is required to build. If you don't have one, you can bypass this by setting the variable to a blank string, however anything using the CurseForge API will not work."
-    ));
+    )));
     let mut config_file =
         config::get_file(cli_app.config_file.unwrap_or_else(config::file_path)).await?;
     let mut config_file_contents = String::new();
@@ -144,22 +151,28 @@ async fn actual_main() -> Result<()> {
         },
         SubCommands::List { verbose } => {
             check_empty_profile(profile)?;
-            for mod_ in &profile.mods {
-                if verbose {
+            if verbose {
+                check_internet().await?;
+                let mut tasks = Vec::new();
+                for mod_ in &profile.mods {
                     use config::structs::ModIdentifier;
-                    check_internet().await?;
                     match &mod_.identifier {
-                        ModIdentifier::CurseForgeProject(project_id) => {
-                            subcommands::list::curseforge(&curseforge, *project_id).await
-                        },
-                        ModIdentifier::ModrinthProject(project_id) => {
-                            subcommands::list::modrinth(&modrinth, project_id).await
-                        },
-                        ModIdentifier::GitHubRepository(full_name) => {
-                            subcommands::list::github(&github, full_name).await
-                        },
-                    }?;
-                } else {
+                        ModIdentifier::CurseForgeProject(project_id) => tasks.push(spawn(
+                            subcommands::list::curseforge(curseforge.clone(), *project_id),
+                        )),
+                        ModIdentifier::ModrinthProject(project_id) => tasks.push(spawn(
+                            subcommands::list::modrinth(modrinth.clone(), project_id.clone()),
+                        )),
+                        ModIdentifier::GitHubRepository(full_name) => tasks.push(spawn(
+                            subcommands::list::github(github.clone(), full_name.clone()),
+                        )),
+                    };
+                }
+                for handle in tasks {
+                    handle.await??;
+                }
+            } else {
+                for mod_ in &profile.mods {
                     println!("{}", mod_.name);
                 }
             }
@@ -200,7 +213,7 @@ async fn actual_main() -> Result<()> {
             check_internet().await?;
             check_empty_profile(profile)?;
             create_dir_all(&profile.output_dir.join(".old")).await?;
-            upgrade(&modrinth, &curseforge, &github, profile).await?;
+            upgrade(modrinth, curseforge, github, profile).await?;
         },
     };
 

src/main.rs

@@ -0,0 +1,18 @@
+use std::sync::{Mutex, MutexGuard};
+
+/// A sketchy way to not deal with mutex poisoning
+///
+/// **WARNING**: If the poison had occured during a write, the data may be corrupted.
+/// _If_ unsafe code had poisoned the mutex, memory corruption is possible
+pub trait MutexExt<T> {
+    fn force_lock(&self) -> MutexGuard<'_, T>;
+}
+
+impl<T> MutexExt<T> for Mutex<T> {
+    fn force_lock(&self) -> MutexGuard<'_, T> {
+        match self.lock() {
+            Ok(guard) => guard,
+            Err(error) => error.into_inner(),
+        }
+    }
+}

src/mutex_ext.rs

@@ -3,8 +3,9 @@ use ferinth::Ferinth;
 use furse::Furse;
 use itertools::Itertools;
 use octocrab::Octocrab;
+use std::sync::Arc;
 
-pub async fn curseforge(curseforge: &Furse, project_id: i32) -> Result<()> {
+pub async fn curseforge(curseforge: Arc<Furse>, project_id: i32) -> Result<()> {
     let project = curseforge.get_mod(project_id).await?;
     let authors = project
         .authors
@@ -41,8 +42,8 @@ pub async fn curseforge(curseforge: &Furse, project_id: i32) -> Result<()> {
     Ok(())
 }
 
-pub async fn modrinth(modrinth: &Ferinth, project_id: &str) -> Result<()> {
-    let project = modrinth.get_project(project_id).await?;
+pub async fn modrinth(modrinth: Arc<Ferinth>, project_id: String) -> Result<()> {
+    let project = modrinth.get_project(&project_id).await?;
     let team_members = modrinth.list_team_members(&project.team).await?;
 
     // Get the usernames of all the developers
@@ -83,7 +84,7 @@ pub async fn modrinth(modrinth: &Ferinth, project_id: &str) -> Result<()> {
 }
 
 /// List all the mods in `profile` with some of their metadata
-pub async fn github(github: &Octocrab, full_name: &(String, String)) -> Result<()> {
+pub async fn github(github: Arc<Octocrab>, full_name: (String, String)) -> Result<()> {
     let repo_handler = github.repos(&full_name.0, &full_name.1);
     let repo = repo_handler.get().await?;
     let releases = repo_handler.releases().list().send().await?;