Add support for additional_remote_packages config

Cargo.toml

@@ -39,6 +39,7 @@ walkdir = "2"
 steamid-ng = "1.0.0"
 godot = { git = "https://github.com/godot-rust/gdext", branch = "master" }
 chrono = "0.4.30"
+url = "2.5.0"
 
 [patch."https://github.com/godot-rust/godot4-prebuilt".godot4-prebuilt]
 git = "https://github.com//godot-rust/godot4-prebuilt"

README.md

@@ -114,6 +114,7 @@ A configuration json file named `config.json` will be located in the `~/.config/
 - hash_check_install - If this parameter is set to true, hash checking will be enabled for each file the engine needs to run, so that if a file has already been extracted/installed, it won't do it again until there is an update to one of the engine files. This is defaulted to false.
 - close_client_on_launch - If this parameter is set to true, the client will close as soon as the game launches, instead of waiting for the engine to complete like normal behavior. This is defaulted to false.
 - steam_app_id_install_wait_in_seconds - How long to wait for the install to complete when installing game dependencies. Defaults to 600.
+- additional_remote_packages - An array of full URLs of additional JSON files to retrieve package information from. If this is provided, the data will be merged with the official package metadata, only being active for new games, and only looking at the games and engines keys.
 
 Logs will be written to file if ```LUX_WRITE_LOGGING=1``` is set. The log file will be located at ```~/.local/state/luxtorpeda/luxtorpeda.log```.
 

src/config.rs

@@ -15,6 +15,7 @@ pub struct Config {
     pub hash_check_install: bool,
     pub close_client_on_launch: bool,
     pub steam_app_id_install_wait_in_seconds: u32,
+    pub additional_remote_packages: Option<Vec<String>>,
 }
 
 impl Default for Config {
@@ -27,6 +28,7 @@ impl Default for Config {
             hash_check_install: true,
             close_client_on_launch: false,
             steam_app_id_install_wait_in_seconds: 600,
+            additional_remote_packages: None,
         }
     }
 }

src/package_metadata.rs

@@ -2,7 +2,9 @@ use log::{error, info};
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::io;
+use std::io::{Error, ErrorKind};
 use std::path::{Path, PathBuf};
+use url::Url;
 
 use crate::config;
 use crate::package;
@@ -181,7 +183,14 @@ impl PackageMetadata {
             info!("packages_json_file exists, reading");
             match fs::read_to_string(packages_json_file) {
                 Ok(s) => match serde_json::from_str::<PackageMetadata>(&s) {
-                    Ok(config) => config,
+                    Ok(metadata) => {
+                        let config = config::Config::from_config_file();
+                        if let Some(remote_packages) = &config.additional_remote_packages {
+                            PackageMetadata::from_remote_packages_cache(metadata, remote_packages)
+                        } else {
+                            metadata
+                        }
+                    }
                     Err(err) => {
                         error!("error parsing packages_json_file: {:?}", err);
                         Default::default()
@@ -198,6 +207,63 @@ impl PackageMetadata {
         }
     }
 
+    pub fn from_remote_packages_cache(
+        mut metadata: PackageMetadata,
+        remote_packages: &[String],
+    ) -> PackageMetadata {
+        for url_str in remote_packages {
+            info!(
+                "from_remote_packages_cache. loading from cache for {}",
+                url_str
+            );
+
+            let parsed_url = match Url::parse(url_str) {
+                Ok(u) => u,
+                Err(err) => {
+                    error!("from_remote_packages_cache. url parse err: {:?}", err);
+                    return metadata;
+                }
+            };
+
+            let filename = match parsed_url.path_segments() {
+                Some(segments) => match segments.last() {
+                    Some(segment) => segment,
+                    None => {
+                        error!("from_remote_packages_cache. url last not found");
+                        return metadata;
+                    }
+                },
+                None => {
+                    error!("from_remote_packages_cache. url path_segments not found");
+                    return metadata;
+                }
+            };
+
+            let local_packages_path =
+                PackageMetadata::path_to_packages_file().with_file_name(filename);
+
+            match fs::read_to_string(local_packages_path) {
+                Ok(s) => match serde_json::from_str::<PackageMetadata>(&s) {
+                    Ok(mut cached_metadata) => {
+                        info!("merging cached remote package of {}", filename);
+                        metadata.games.append(&mut cached_metadata.games);
+                        metadata.engines.append(&mut cached_metadata.engines);
+                    }
+                    Err(err) => {
+                        error!("error parsing from_remote_packages_cache: {:?}", err);
+                        return metadata;
+                    }
+                },
+                Err(err) => {
+                    error!("error reading from_remote_packages_cache: {:?}", err);
+                    return metadata;
+                }
+            }
+        }
+
+        metadata
+    }
+
     pub fn find_game_by_app_id(&self, app_id: &str) -> Option<Game> {
         self.games.iter().find(|x| x.app_id == app_id).cloned()
     }
@@ -238,7 +304,7 @@ impl PackageMetadata {
     pub fn update_packages_json() -> io::Result<()> {
         let config = config::Config::from_config_file();
         if !config.should_do_update {
-            return Ok(());
+            return PackageMetadata::download_additional_remote_packages(&config);
         }
 
         let packages_json_file = PackageMetadata::path_to_packages_file();
@@ -314,6 +380,73 @@ impl PackageMetadata {
             }
         }
 
+        PackageMetadata::download_additional_remote_packages(&config)
+    }
+
+    pub fn download_additional_remote_packages(config: &config::Config) -> io::Result<()> {
+        if let Some(additional_remote_packages) = &config.additional_remote_packages {
+            for url_str in additional_remote_packages {
+                info!(
+                    "download_additional_remote_packages. downloading from {}",
+                    url_str
+                );
+
+                let parsed_url = match Url::parse(url_str) {
+                    Ok(u) => u,
+                    Err(err) => {
+                        let error_str = format!(
+                            "download_additional_remote_packages. url parse err: {:?}",
+                            err
+                        );
+                        error!("{}", error_str);
+                        return Err(Error::new(ErrorKind::Other, error_str));
+                    }
+                };
+
+                let filename = match parsed_url.path_segments() {
+                    Some(segments) => match segments.last() {
+                        Some(segment) => segment,
+                        None => {
+                            let error_str =
+                                "download_additional_remote_packages. url last not found";
+                            error!("{}", error_str);
+                            return Err(Error::new(ErrorKind::Other, error_str));
+                        }
+                    },
+                    None => {
+                        let error_str =
+                            "download_additional_remote_packages. url path_segments not found";
+                        error!("{}", error_str);
+                        return Err(Error::new(ErrorKind::Other, error_str));
+                    }
+                };
+
+                let local_packages_path =
+                    PackageMetadata::path_to_packages_file().with_file_name(filename);
+
+                match reqwest::blocking::get(url_str) {
+                    Ok(mut response) => {
+                        let mut dest = fs::File::create(&local_packages_path)?;
+                        io::copy(&mut response, &mut dest)?;
+                        info!(
+                            "download_additional_remote_packages {} is saved to {:?}",
+                            url_str, local_packages_path
+                        );
+                    }
+                    Err(err) => {
+                        let error_str = format!(
+                            "download_additional_remote_packages. download err: {:?}",
+                            err
+                        );
+                        error!("{}", error_str);
+                        return Err(Error::new(ErrorKind::Other, error_str));
+                    }
+                }
+            }
+        } else {
+            info!("download_additional_remote_packages, no remote packages list given");
+        }
+
         Ok(())
     }