From 76e58f780dc5c5c370cf3cb4a7144b6b12a71522 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 08:10:54 -0800 Subject: [PATCH 01/13] added written to --- tests/data/files.json | 15 ++++++++++----- tests/data/saved.json | 9 ++++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/data/files.json b/tests/data/files.json index 1fa14ac..2f0d919 100644 --- a/tests/data/files.json +++ b/tests/data/files.json @@ -5,7 +5,8 @@ "hash": "", "len": 186, "modified": 1700331937339576, - "last_saved": null + "last_saved": null, + "written_to": [] }, { "key": "IzEy7mr29Z6FCePS", @@ -13,7 +14,8 @@ "hash": "", "len": 19, "modified": 1700331937339576, - "last_saved": null + "last_saved": null, + "written_to": [] }, { "key": "Npeu7mr2B2ua25Sn", @@ -21,7 +23,8 @@ "hash": "", "len": 474707, "modified": 1699990650530503, - "last_saved": null + "last_saved": null, + "written_to": [] }, { "key": "0bJk7mr2C8SxJnKA", @@ -29,7 +32,8 @@ "hash": "", "len": 186, "modified": 1700331937339576, - "last_saved": null + "last_saved": null, + "written_to": [] }, { "key": "8iwl7mr2DU3XnMkT", @@ -37,6 +41,7 @@ "hash": "", "len": 19, "modified": 1700331937339576, - "last_saved": null + "last_saved": null, + "written_to": [] } ] diff --git a/tests/data/saved.json b/tests/data/saved.json index bf73771..44fbc68 100644 --- a/tests/data/saved.json +++ b/tests/data/saved.json @@ -5,7 +5,8 @@ "hash": "", "len": 186, "modified": 1700331937339576, - "last_saved": "2023-11-19T15:19:53.423437223" + "last_saved": "2023-11-19T15:19:53.423437223", + "written_to": [] }, { "key": "IzEy7mr29Z6FCePS", @@ -13,7 +14,8 @@ "hash": "", "len": 19, "modified": 1700331937339576, - "last_saved": "2023-11-19T15:19:53.423437223" + "last_saved": "2023-11-19T15:19:53.423437223", + "written_to": [] }, { "key": "8iwl7mr2DU3XnMkT", @@ -21,7 +23,8 @@ "hash": "", "len": 18, "modified": 1700331937339576, - "last_saved": "2023-11-19T15:19:53.423437223" + "last_saved": "2023-11-19T15:19:53.423437223", + "written_to": [] } ] From f2d74bfae3d6ec25268f351116e29097b33808e7 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 08:11:56 -0800 Subject: [PATCH 02/13] added written to to FileModel --- src/backup_queue.rs | 2 ++ src/file_model.rs | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/backup_queue.rs b/src/backup_queue.rs index f48be7e..4422626 100644 --- a/src/backup_queue.rs +++ b/src/backup_queue.rs @@ -77,6 +77,8 @@ impl BackupQueue { pub fn match_files(&self, ref_model: &FileModel, target_path: &Path) -> Option { let filename = target_path.to_str().unwrap(); let mut target_model = FileModel::new(filename); + // this ensures that the last_updated gets set to the correct record + target_model.key = ref_model.key.clone(); if target_path.exists() { target_model = target_model.read_metadata().unwrap(); diff --git a/src/file_model.rs b/src/file_model.rs index 50b904b..f8b2d25 100644 --- a/src/file_model.rs +++ b/src/file_model.rs @@ -4,7 +4,7 @@ /// use anyhow::{anyhow, Result}; use chrono::naive::NaiveDateTime; -use hashbrown::HashMap; +use hashbrown::{HashMap, HashSet}; use log::{error, info, warn}; use openssl::sha; use serde::{Deserialize, Serialize}; @@ -14,7 +14,7 @@ use std::io::{BufReader, Read, Write}; use std::path::PathBuf; use domain_keys::keys::RouteKey; -#[derive(Debug, Default, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Default, Clone, Deserialize, Serialize, PartialEq, Eq)] pub struct FileModel { pub key: String, pub path: PathBuf, @@ -22,6 +22,7 @@ pub struct FileModel { pub len: u64, pub modified: u64, pub last_saved: Option, + pub written_to: HashSet, } impl FileModel { @@ -33,6 +34,7 @@ impl FileModel { len: 0, modified: 0, last_saved: None, + written_to: HashSet::new(), } } @@ -44,6 +46,7 @@ impl FileModel { len, modified, last_saved: None, + written_to: HashSet::new(), } } @@ -56,6 +59,7 @@ impl FileModel { len: model.len, modified: model.modified, last_saved: model.last_saved, + written_to: model.written_to, } } From 624b03a03eaa4c14bc3f1a8db846a26d920f1bdd Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 08:12:58 -0800 Subject: [PATCH 03/13] version bump --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5f6639..8c36dd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -884,7 +884,7 @@ dependencies = [ [[package]] name = "replica" -version = "0.3.8" +version = "0.3.9" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 444a3c5..1e4cbcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "replica" -version = "0.3.8" +version = "0.3.9" edition = "2021" authors = ["darryl.west@raincitysoftware.com"] rust-version = "1.70" From 967b5c2c7ed97ecaf65e2462d9dbf7da2c6c1ea2 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 09:19:56 -0800 Subject: [PATCH 04/13] refactored backup queue to backup process --- Cargo.lock | 68 ++++++++++++++++++---- Cargo.toml | 6 +- src/{backup_queue.rs => backup_process.rs} | 24 ++++---- src/bin/replica.rs | 4 +- src/file_model.rs | 2 +- src/lib.rs | 2 +- 6 files changed, 75 insertions(+), 31 deletions(-) rename src/{backup_queue.rs => backup_process.rs} (92%) diff --git a/Cargo.lock b/Cargo.lock index 8c36dd7..bcadefc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -420,20 +426,15 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash", + "allocator-api2", "serde", ] -[[package]] -name = "hashbrown" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" - [[package]] name = "heck" version = "0.4.1" @@ -884,13 +885,13 @@ dependencies = [ [[package]] name = "replica" -version = "0.3.9" +version = "0.4.1" dependencies = [ "anyhow", "chrono", "clap", "domain_keys", - "hashbrown 0.13.2", + "hashbrown 0.14.2", "hex", "log", "log4rs", @@ -1055,6 +1056,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1272,11 +1282,36 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" dependencies = [ "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap 2.1.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", ] [[package]] @@ -1585,6 +1620,15 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winnow" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/Cargo.toml b/Cargo.toml index 1e4cbcf..95cffc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "replica" -version = "0.3.9" +version = "0.4.1" edition = "2021" authors = ["darryl.west@raincitysoftware.com"] rust-version = "1.70" @@ -24,9 +24,9 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_derive = "1.0" chrono = { version = "0.4", features = ["serde"] } -toml = "0.5.9" +toml = "0.8.8" reqwest = { version = "0.11", features = ["json"] } -hashbrown = { version = "0.13.1", features = ["serde"] } +hashbrown = { version = "0.14.2", features = ["serde"] } openssl = "0.10.43" hex = "0.4.3" walkdir = "2.3.2" diff --git a/src/backup_queue.rs b/src/backup_process.rs similarity index 92% rename from src/backup_queue.rs rename to src/backup_process.rs index 4422626..abd636a 100644 --- a/src/backup_queue.rs +++ b/src/backup_process.rs @@ -1,6 +1,6 @@ /// Backup Queue to save queued files to targets /// -/// # Backup Queue +/// # Backup Process /// /// create with target folder and queue vector; return the list of saved files updated with save date /// @@ -11,14 +11,14 @@ use log::{debug, error, info}; use std::fs; use std::path::{Path, PathBuf}; -pub struct BackupQueue { +pub struct BackupProcess { pub target: PathBuf, pub files: Vec, pub dryrun: bool, } -impl BackupQueue { - pub fn new(path: &str, files: Vec, dryrun: bool) -> BackupQueue { +impl BackupProcess { + pub fn new(path: &str, files: Vec, dryrun: bool) -> BackupProcess { let mut tp = path.to_string(); if !tp.ends_with('/') { tp.push('/'); @@ -26,7 +26,7 @@ impl BackupQueue { info!("dryrun = {}", dryrun); - BackupQueue { + BackupProcess { target: PathBuf::from(tp), files, dryrun, @@ -157,7 +157,7 @@ mod tests { let files = create_filelist(); let flen = files.len(); - let backup = BackupQueue::new(path, files, true); + let backup = BackupProcess::new(path, files, true); assert_eq!(backup.files.len(), flen); } @@ -167,7 +167,7 @@ mod tests { let files = create_filelist(); let flen = files.len(); - let backup = BackupQueue::new(path, files, true); + let backup = BackupProcess::new(path, files, true); assert_eq!(flen, backup.files.len()); assert!(true); @@ -179,7 +179,7 @@ mod tests { let dest = FileModel::new("tests/tback/file3.txt"); println!("src: {}, dest: {:?}", src.display(), dest); - let backup = BackupQueue::new("./", vec![], false); + let backup = BackupProcess::new("./", vec![], false); let response = backup.copy_model(src, dest); println!("{:?}", response); @@ -192,7 +192,7 @@ mod tests { let dest = FileModel::new("tests/tback/file-nofile.txt"); println!("src: {}, dest: {:?}", src.display(), dest); - let backup = BackupQueue::new("./", vec![], false); + let backup = BackupProcess::new("./", vec![], false); let response = backup.copy_model(src, dest); println!("{:?}", response); @@ -206,7 +206,7 @@ mod tests { println!("src: {}, dest: {:?}", src.display(), dest.display()); - let backup = BackupQueue::new("./", vec![], false); + let backup = BackupProcess::new("./", vec![], false); let response = backup.copy(src, dest); println!("{:?}", response); @@ -220,7 +220,7 @@ mod tests { let dest = Path::new("tests/tback/file2.txt"); println!("src: {:?}, dest: {}", src, dest.display()); - let backup = BackupQueue::new("./", vec![], true); + let backup = BackupProcess::new("./", vec![], true); let response = backup.match_files(&src, dest); println!("{:?}", response); @@ -234,7 +234,7 @@ mod tests { let dest = Path::new("tests/tback/file1.txt"); println!("src: {:?}, dest: {}", src, dest.display()); - let backup = BackupQueue::new("./", vec![], true); + let backup = BackupProcess::new("./", vec![], true); let response = backup.match_files(&src, dest); println!("{:?}", response); diff --git a/src/bin/replica.rs b/src/bin/replica.rs index aa7f5ce..b72fdfe 100644 --- a/src/bin/replica.rs +++ b/src/bin/replica.rs @@ -4,7 +4,7 @@ use anyhow::Result; use clap::Parser; use log::{error, info, warn}; -use replica::backup_queue::BackupQueue; +use replica::backup_process::BackupProcess; use replica::config::Config; use replica::file_model::FileModel; use replica::file_walker::FileWalker; @@ -58,7 +58,7 @@ fn run(cli: Cli) -> Result<()> { } let target_dir = &config.targets[0]; - let backup = BackupQueue::new(target_dir.as_str(), files.clone(), cli.dryrun); + let backup = BackupProcess::new(target_dir.as_str(), files.clone(), cli.dryrun); let results = backup.process(); if results.is_ok() { let saved_list = results.unwrap(); diff --git a/src/file_model.rs b/src/file_model.rs index f8b2d25..57753a1 100644 --- a/src/file_model.rs +++ b/src/file_model.rs @@ -4,6 +4,7 @@ /// use anyhow::{anyhow, Result}; use chrono::naive::NaiveDateTime; +use domain_keys::keys::RouteKey; use hashbrown::{HashMap, HashSet}; use log::{error, info, warn}; use openssl::sha; @@ -12,7 +13,6 @@ use std::env; use std::fs::File; use std::io::{BufReader, Read, Write}; use std::path::PathBuf; -use domain_keys::keys::RouteKey; #[derive(Debug, Default, Clone, Deserialize, Serialize, PartialEq, Eq)] pub struct FileModel { diff --git a/src/lib.rs b/src/lib.rs index a2feb35..bd65f7e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ #![doc = include_str!("../README.md")] -pub mod backup_queue; +pub mod backup_process; pub mod config; pub mod file_model; pub mod file_walker; From 600e5fc3728a44ee7b8e7f9200ace78849f62b76 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 10:11:39 -0800 Subject: [PATCH 05/13] refactor to merge new data with db (still buggy) --- src/backup_process.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/backup_process.rs b/src/backup_process.rs index abd636a..566df3e 100644 --- a/src/backup_process.rs +++ b/src/backup_process.rs @@ -67,7 +67,7 @@ impl BackupProcess { let target_model = target_model.unwrap(); - match self.copy_model(model.path.as_path(), target_model) { + match self.copy_model(model, target_model) { Ok(model) => Some(model), Err(_e) => None, } @@ -94,24 +94,29 @@ impl BackupProcess { Some(target_model) } - /// copy the source to destination and return the updated model - pub fn copy_model(&self, src: &Path, dest: FileModel) -> Result { + /// Copy the source to destination; update the source last_saved date and written to hash; + /// Return the updated src model + pub fn copy_model(&self, src: &FileModel, dest: FileModel) -> Result { let save_model = FileModel::copy_from(dest); if self.dryrun { return Ok(save_model); } + let src_path = src.path.as_path(); let dest_path = save_model.path.as_path(); - if self.copy(src, dest_path).is_err() { + if self.copy(src_path, dest_path).is_err() { let msg = format!("error saving to: {}", dest_path.display()); error!("{}", msg); return Err(anyhow!("{}", msg)); } - let mut model = save_model.read_metadata()?; + let mut model = FileModel::copy_from(src.to_owned()); let now = Utc::now().naive_utc(); + let write_path = dest_path.to_str().unwrap(); + model.last_saved = Some(now); + model.written_to.insert(write_path.to_string()); info!("saved: {:?}", model); @@ -175,12 +180,12 @@ mod tests { #[test] fn copy_model() { - let src = Path::new("tests/file3.txt"); + let src = FileModel::new("tests/file3.txt"); let dest = FileModel::new("tests/tback/file3.txt"); - println!("src: {}, dest: {:?}", src.display(), dest); + println!("src: {:?}, dest: {:?}", src, dest); let backup = BackupProcess::new("./", vec![], false); - let response = backup.copy_model(src, dest); + let response = backup.copy_model(&src, dest); println!("{:?}", response); assert!(response.is_ok()); @@ -188,12 +193,12 @@ mod tests { #[test] fn bad_copy_model() { - let src = Path::new("tests/file-nofile.txt"); + let src = FileModel::new("tests/file-nofile.txt"); let dest = FileModel::new("tests/tback/file-nofile.txt"); - println!("src: {}, dest: {:?}", src.display(), dest); + println!("src: {:?}, dest: {:?}", src, dest); let backup = BackupProcess::new("./", vec![], false); - let response = backup.copy_model(src, dest); + let response = backup.copy_model(&src, dest); println!("{:?}", response); assert!(response.is_err()); From edaac2f39c782cc503fbc17a6f6f864c38ed23b5 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 12:05:40 -0800 Subject: [PATCH 06/13] added db ops k/v store --- src/bin/replica.rs | 9 ++---- src/db_ops.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 3 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 src/db_ops.rs diff --git a/src/bin/replica.rs b/src/bin/replica.rs index b72fdfe..3f3d14d 100644 --- a/src/bin/replica.rs +++ b/src/bin/replica.rs @@ -34,7 +34,7 @@ fn cd_app_home(app_home: &str) { env::set_current_dir(app_home).unwrap_or_else(|_| panic!("{}", msg)); } -/// TODO: refactor this to multiple methods +/// the primary process fn run(cli: Cli) -> Result<()> { let start_time = Instant::now(); let config = Config::read_config(cli.config.as_str())?; @@ -48,14 +48,11 @@ fn run(cli: Cli) -> Result<()> { warn!("THIS IS A DRY RUN!"); } + // read the current database DbOps + let walker = FileWalker::new(config.clone()); if let Ok(files) = walker.walk_files_and_folders() { info!("file count: {}", files.len()); - // now compare and update if necessary - match FileModel::write_dbfile(&config.dbfile, files.clone()) { - Ok(()) => info!("file model list written to {}", config.dbfile), - Err(e) => error!("error: {}, writing file model list to {}", e, config.dbfile), - } let target_dir = &config.targets[0]; let backup = BackupProcess::new(target_dir.as_str(), files.clone(), cli.dryrun); diff --git a/src/db_ops.rs b/src/db_ops.rs new file mode 100644 index 0000000..9c842c5 --- /dev/null +++ b/src/db_ops.rs @@ -0,0 +1,81 @@ +/// DbOps - database operations + +use anyhow::{anyhow, Result}; +use hashbrown::HashMap; +use std::path::PathBuf; +use crate::file_model::FileModel; +use log::{info, warn, error}; +use std::fs::File; +use std::io::{BufReader, Read}; +// use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Clone)] +pub struct DbOps { + dbpath: PathBuf, + db: HashMap, + index: HashMap, +} + +impl DbOps { + /// initializes the database ; reads the dbfile, stores in k/v and creates index. + pub fn init(dbpath: PathBuf) -> Result { + let mut client = DbOps { + dbpath, + db: HashMap::new(), + index: HashMap::new(), + }; + + match client.read_dbfile() { + Ok(_) => Ok(client), + Err(e) => { + let msg = format!("error initializing database: {}", e); + error!("{}", msg); + Err(anyhow!("{}", msg)) + } + } + } + + fn read_dbfile(&mut self) -> Result<()> { + info!("read database file from {}", self.dbpath.display()); + + let file = match File::open(self.dbpath.as_os_str()) { + Ok(file) => file, + Err(e) => { + warn!("New empty list: {}", e); + return Ok(()); + } + }; + + let mut reader = BufReader::new(file); + + let mut text = String::new(); + reader.read_to_string(&mut text)?; + + let list: Vec = serde_json::from_str(&text)?; + + for model in list { + let key = model.key.to_string(); + let mpath = model.path.to_str().unwrap(); + + self.db.insert(key.clone(), model.clone()); + self.index.insert(mpath.to_string(), key); + } + + + Ok(()) + } + +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn init() { + let filename = "tests/data/files.json"; + let result = DbOps::init(PathBuf::from(filename)); + + println!("{:?}", result); + } +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index bd65f7e..4e75d12 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod backup_process; pub mod config; pub mod file_model; pub mod file_walker; +pub mod db_ops; /// The current version as read from the cargo toml file /// From 4bd42495e4f80b9898ce8db02baf3675bf2d036d Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 12:17:57 -0800 Subject: [PATCH 07/13] unit tests --- src/{db_ops.rs => kv_store.rs} | 29 +++++++++++++++++++++++------ src/lib.rs | 2 +- 2 files changed, 24 insertions(+), 7 deletions(-) rename src/{db_ops.rs => kv_store.rs} (74%) diff --git a/src/db_ops.rs b/src/kv_store.rs similarity index 74% rename from src/db_ops.rs rename to src/kv_store.rs index 9c842c5..3e1be91 100644 --- a/src/db_ops.rs +++ b/src/kv_store.rs @@ -1,4 +1,4 @@ -/// DbOps - database operations +/// Key/Value Store - database operations use anyhow::{anyhow, Result}; use hashbrown::HashMap; @@ -10,16 +10,16 @@ use std::io::{BufReader, Read}; // use serde::{Deserialize, Serialize}; #[derive(Debug, Default, Clone)] -pub struct DbOps { +pub struct KvStore { dbpath: PathBuf, db: HashMap, index: HashMap, } -impl DbOps { +impl KvStore { /// initializes the database ; reads the dbfile, stores in k/v and creates index. - pub fn init(dbpath: PathBuf) -> Result { - let mut client = DbOps { + pub fn init(dbpath: PathBuf) -> Result { + let mut client = KvStore { dbpath, db: HashMap::new(), index: HashMap::new(), @@ -74,8 +74,25 @@ mod tests { #[test] fn init() { let filename = "tests/data/files.json"; - let result = DbOps::init(PathBuf::from(filename)); + let result = KvStore::init(PathBuf::from(filename)); println!("{:?}", result); + assert!(result.is_ok()); + } + + #[test] + fn init_nofile() { + let result = KvStore::init(PathBuf::from("tests/notarealfile.json")); + + println!("{:?}", result); + assert!(result.is_ok()); + } + + #[test] + fn init_bad_data() { + let result = KvStore::init(PathBuf::from("tests/file1.txt")); + + println!("{:?}", result); + assert!(result.is_err()); } } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 4e75d12..1cca2ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ pub mod backup_process; pub mod config; pub mod file_model; pub mod file_walker; -pub mod db_ops; +pub mod kv_store; /// The current version as read from the cargo toml file /// From 5a0958f2dc6b6c8f0d2fb6e1bce343114286d850 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 12:19:24 -0800 Subject: [PATCH 08/13] clean ups --- src/kv_store.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/kv_store.rs b/src/kv_store.rs index 3e1be91..7e7cee0 100644 --- a/src/kv_store.rs +++ b/src/kv_store.rs @@ -1,12 +1,11 @@ +use crate::file_model::FileModel; /// Key/Value Store - database operations - use anyhow::{anyhow, Result}; use hashbrown::HashMap; -use std::path::PathBuf; -use crate::file_model::FileModel; -use log::{info, warn, error}; +use log::{error, info, warn}; use std::fs::File; use std::io::{BufReader, Read}; +use std::path::PathBuf; // use serde::{Deserialize, Serialize}; #[derive(Debug, Default, Clone)] @@ -37,7 +36,7 @@ impl KvStore { fn read_dbfile(&mut self) -> Result<()> { info!("read database file from {}", self.dbpath.display()); - + let file = match File::open(self.dbpath.as_os_str()) { Ok(file) => file, Err(e) => { @@ -61,10 +60,8 @@ impl KvStore { self.index.insert(mpath.to_string(), key); } - Ok(()) } - } #[cfg(test)] @@ -95,4 +92,4 @@ mod tests { println!("{:?}", result); assert!(result.is_err()); } -} \ No newline at end of file +} From 21692b77a9383e85ef118a8e408650149a00c1bb Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 13:01:14 -0800 Subject: [PATCH 09/13] db api get set len is dirty --- src/kv_store.rs | 59 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/kv_store.rs b/src/kv_store.rs index 7e7cee0..67d4a1c 100644 --- a/src/kv_store.rs +++ b/src/kv_store.rs @@ -13,6 +13,7 @@ pub struct KvStore { dbpath: PathBuf, db: HashMap, index: HashMap, + dirty_flag: bool, } impl KvStore { @@ -22,6 +23,7 @@ impl KvStore { dbpath, db: HashMap::new(), index: HashMap::new(), + dirty_flag: false, }; match client.read_dbfile() { @@ -62,12 +64,69 @@ impl KvStore { Ok(()) } + + /// get the file model or return None if it doesn't exist + pub fn get(&self, key: &str) -> Option<&FileModel> { + self.db.get(key) + } + + /// insert the model into k/v store's db + pub fn set(&mut self, model: FileModel) -> Result<()> { + self.dirty_flag = true; + let key = model.key.to_string(); + let _ = self.db.insert(key, model); + + Ok(()) + } + + /// return the size of this database + pub fn len(&self) -> usize { + self.db.len() + } + + /// return true if the data has been updated, else false + pub fn is_dirty(&self) -> bool { + self.dirty_flag + } } #[cfg(test)] mod tests { use super::*; + #[test] + fn getset_dirty_len() { + let filename = "tests/data/files.json"; + let mut client = KvStore::init(PathBuf::from(filename)).unwrap(); + assert!(!client.is_dirty()); + let count: usize = 5; + assert_eq!(client.len(), count); + + let result = client.get("bad-key"); + assert!(result.is_none()); + + let result = client.get("8iwl7mr2DU3XnMkT"); + println!("{:?}", result); + assert!(result.is_some()); + let model = result.unwrap(); + assert_eq!(model.len, 19); + assert!(!client.is_dirty()); + + let mut model = model.clone(); + let myhash = "1234567"; + model.hash = myhash.to_string(); + assert_eq!(&model.hash, &myhash); + + let result = client.set(model.clone()); + assert!(result.is_ok()); + assert!(client.is_dirty()); + + let updated = client.get(&model.key).unwrap(); + println!("up: {:?}", updated); + assert_eq!(updated.hash, myhash); + assert_eq!(client.len(), count); + } + #[test] fn init() { let filename = "tests/data/files.json"; From ad4fc9148e849a020e1a4d63ad0f14d162ac1a58 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 14:24:27 -0800 Subject: [PATCH 10/13] updates to savedb --- src/kv_store.rs | 73 ++++++++++++++++++++++++++++++++++++++++++------ tests/.gitignore | 1 + 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/src/kv_store.rs b/src/kv_store.rs index 67d4a1c..28bfc26 100644 --- a/src/kv_store.rs +++ b/src/kv_store.rs @@ -4,22 +4,22 @@ use anyhow::{anyhow, Result}; use hashbrown::HashMap; use log::{error, info, warn}; use std::fs::File; -use std::io::{BufReader, Read}; +use std::io::{BufReader, Read, Write}; use std::path::PathBuf; // use serde::{Deserialize, Serialize}; #[derive(Debug, Default, Clone)] -pub struct KvStore { +pub struct KeyValueStore { dbpath: PathBuf, db: HashMap, index: HashMap, dirty_flag: bool, } -impl KvStore { +impl KeyValueStore { /// initializes the database ; reads the dbfile, stores in k/v and creates index. - pub fn init(dbpath: PathBuf) -> Result { - let mut client = KvStore { + pub fn init(dbpath: PathBuf) -> Result { + let mut client = KeyValueStore { dbpath, db: HashMap::new(), index: HashMap::new(), @@ -88,16 +88,71 @@ impl KvStore { pub fn is_dirty(&self) -> bool { self.dirty_flag } + + /// find the file model from the path + pub fn find(&self, path: &str) -> Option<&FileModel> { + let key = self.index.get(path); + if key.is_none() { + return None; + } + + self.db.get(key.unwrap()) + } + + /// save the kv to file + pub fn savedb(&mut self, filename: &str) -> Result<()> { + info!("save the k/v models as a list to file: {}", filename); + let list: Vec = self.db.clone().into_values().collect(); + let json = serde_json::to_string_pretty(&list).unwrap(); + + match File::create(filename) { + Ok(mut buf) => buf.write_all(json.as_bytes())?, + Err(e) => { + let msg = format!("dbfile write error: {}, {}", filename, e); + error!("{}", msg); + return Err(anyhow!("{}", msg)); + } + } + + info!("reset the dirty flag to false"); + self.dirty_flag = false; + + Ok(()) + } } #[cfg(test)] mod tests { use super::*; + #[test] + fn savedb() { + let filename = "tests/data/files.json"; + let mut client = KeyValueStore::init(PathBuf::from(filename)).unwrap(); + + let filename = "tests/data/backup.json"; + let result = client.savedb(filename); + assert!(result.is_ok()) + } + + #[test] + fn find() { + let filename = "tests/data/files.json"; + let client = KeyValueStore::init(PathBuf::from(filename)).unwrap(); + + let model = client.find("a/bad/path"); + assert!(model.is_none()); + + let model = client.find("./tests/big-file.pdf"); + assert!(model.is_some()); + let model = model.unwrap(); + assert_eq!(model.key, "Npeu7mr2B2ua25Sn"); + } + #[test] fn getset_dirty_len() { let filename = "tests/data/files.json"; - let mut client = KvStore::init(PathBuf::from(filename)).unwrap(); + let mut client = KeyValueStore::init(PathBuf::from(filename)).unwrap(); assert!(!client.is_dirty()); let count: usize = 5; assert_eq!(client.len(), count); @@ -130,7 +185,7 @@ mod tests { #[test] fn init() { let filename = "tests/data/files.json"; - let result = KvStore::init(PathBuf::from(filename)); + let result = KeyValueStore::init(PathBuf::from(filename)); println!("{:?}", result); assert!(result.is_ok()); @@ -138,7 +193,7 @@ mod tests { #[test] fn init_nofile() { - let result = KvStore::init(PathBuf::from("tests/notarealfile.json")); + let result = KeyValueStore::init(PathBuf::from("tests/notarealfile.json")); println!("{:?}", result); assert!(result.is_ok()); @@ -146,7 +201,7 @@ mod tests { #[test] fn init_bad_data() { - let result = KvStore::init(PathBuf::from("tests/file1.txt")); + let result = KeyValueStore::init(PathBuf::from("tests/file1.txt")); println!("{:?}", result); assert!(result.is_err()); diff --git a/tests/.gitignore b/tests/.gitignore index b71fd35..9174884 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,2 @@ tback-tmp +data/backup.json From 4997f4d4f5caa8393024e93a51eafc462eb59045 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 16:53:01 -0800 Subject: [PATCH 11/13] refactored to use kv database --- .gitignore | 2 +- .test-replica/config/run-config.toml | 3 +- .test-replica/config/run1-config.toml | 15 ++++ .test-replica/data/run1-file.json | 47 +++++++++++ .test-replica/data/run2-file.json | 47 +++++++++++ justfile | 8 +- src/backup_process.rs | 39 +++++---- src/bin/replica.rs | 105 ++++++++++++++++++----- src/file_model.rs | 117 +++----------------------- src/kv_store.rs | 22 +++-- tests/changed-file.txt | 1 + 11 files changed, 246 insertions(+), 160 deletions(-) create mode 100644 .test-replica/config/run1-config.toml create mode 100644 .test-replica/data/run1-file.json create mode 100644 .test-replica/data/run2-file.json create mode 100644 tests/changed-file.txt diff --git a/.gitignore b/.gitignore index f9281d8..6ddccb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ /target coverage - +tback diff --git a/.test-replica/config/run-config.toml b/.test-replica/config/run-config.toml index bad0891..ad64021 100644 --- a/.test-replica/config/run-config.toml +++ b/.test-replica/config/run-config.toml @@ -8,7 +8,8 @@ files = [ "tests/file1.txt", "tests/file2.txt", "tests/big-file.pdf", + "tests/changed-file.txt", ] -dbfile = ".test-replica/data/files.json" +dbfile = ".test-replica/data/run2-file.json" compress = false encrypt = false diff --git a/.test-replica/config/run1-config.toml b/.test-replica/config/run1-config.toml new file mode 100644 index 0000000..381f48d --- /dev/null +++ b/.test-replica/config/run1-config.toml @@ -0,0 +1,15 @@ +name = "test-run-replica" +version = "0.1.1" +home = "./" +logging_config = ".test-replica/config/console.yaml" +targets = ["tback"] +source_folders = [ "tests/.config" ] +files = [ + "tests/file1.txt", + "tests/file2.txt", + "tests/big-file.pdf", + "tests/changed-file.txt", +] +dbfile = ".test-replica/data/run1-file.json" +compress = false +encrypt = false diff --git a/.test-replica/data/run1-file.json b/.test-replica/data/run1-file.json new file mode 100644 index 0000000..3d8968d --- /dev/null +++ b/.test-replica/data/run1-file.json @@ -0,0 +1,47 @@ +[ + { + "key": "gfl27msaerQYZ1Vy", + "path": "tback/./tests/.config/file1.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "9x2h7msaerQZU733", + "path": "tback/./tests/.config/file2.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "p4qg7msaerQPepHa", + "path": "tback/./tests/big-file.pdf", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "XuZ57msaerQLJtwC", + "path": "tback/./tests/file1.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "sSzd7msaerQPXcfy", + "path": "tback/./tests/file2.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + } +] \ No newline at end of file diff --git a/.test-replica/data/run2-file.json b/.test-replica/data/run2-file.json new file mode 100644 index 0000000..3d8968d --- /dev/null +++ b/.test-replica/data/run2-file.json @@ -0,0 +1,47 @@ +[ + { + "key": "gfl27msaerQYZ1Vy", + "path": "tback/./tests/.config/file1.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "9x2h7msaerQZU733", + "path": "tback/./tests/.config/file2.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "p4qg7msaerQPepHa", + "path": "tback/./tests/big-file.pdf", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "XuZ57msaerQLJtwC", + "path": "tback/./tests/file1.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + }, + { + "key": "sSzd7msaerQPXcfy", + "path": "tback/./tests/file2.txt", + "hash": "", + "len": 0, + "modified": 0, + "last_saved": null, + "written_to": [] + } +] \ No newline at end of file diff --git a/justfile b/justfile index f760fcc..bd1557c 100644 --- a/justfile +++ b/justfile @@ -13,18 +13,18 @@ os := `uname` # run the standard tests test: clear - /bin/rm -fr tests/tback-tmp + /bin/rm -fr tests/tback-tmp tback cargo test # run the standard tests + clippy and fmt test-all: clear - /bin/rm -fr tests/tback-tmp + /bin/rm -fr tests/tback-tmp tback cargo test -- --include-ignored && just format # clean the project clean: - /bin/rm -fr tests/tback-tmp + /bin/rm -fr tests/tback-tmp tback cargo clean # build the debug target @@ -55,7 +55,7 @@ watch: # cover - runs code test coverage report and writes to coverage folder cover: - /bin/rm -fr tests/tback-tmp + /bin/rm -fr tests/tback-tmp tback cargo tarpaulin --out html --output-dir coverage && mv coverage/tarpaulin-report.html coverage/index.html # start a http server in the coverage folder diff --git a/src/backup_process.rs b/src/backup_process.rs index 566df3e..4367959 100644 --- a/src/backup_process.rs +++ b/src/backup_process.rs @@ -5,6 +5,7 @@ /// create with target folder and queue vector; return the list of saved files updated with save date /// use crate::file_model::FileModel; +use crate::kv_store::KeyValueStore; use anyhow::{anyhow, Result}; use chrono::Utc; use log::{debug, error, info}; @@ -34,24 +35,29 @@ impl BackupProcess { } /// process the file list; return the list of files that were backup - pub fn process(&self) -> Result> { + pub fn process(&self, mut db: KeyValueStore) -> Result<()> { info!("process the backup queue"); - let mut saved: Vec = Vec::new(); let files = self.files.clone(); for file_model in files { let fpath = file_model.path.as_os_str(); match self.check_and_copy_file(&file_model) { - Some(backup_model) => { - info!("backup: {:?} -> {}", fpath, backup_model.path.display()); - - saved.push(backup_model); + Some(saved_model) => { + info!("file backup: {:?} -> {}", fpath, saved_model.path.display()); + + // save to db + let resp = db.set(saved_model.clone()); + if resp.is_err() { + error!("could not save to database: {:?}", resp); + } else { + info!("saved to db: {:?}", saved_model); + } } None => debug!("skip {:?}", fpath), } } - Ok(saved) + Ok(()) } /// create the target path; check stat to see backup is required @@ -108,19 +114,18 @@ impl BackupProcess { if self.copy(src_path, dest_path).is_err() { let msg = format!("error saving to: {}", dest_path.display()); error!("{}", msg); - return Err(anyhow!("{}", msg)); - } - - let mut model = FileModel::copy_from(src.to_owned()); - let now = Utc::now().naive_utc(); - let write_path = dest_path.to_str().unwrap(); + Err(anyhow!("{}", msg)) + } else { + let now = Utc::now().naive_utc(); + let write_path = dest_path.to_str().unwrap(); - model.last_saved = Some(now); - model.written_to.insert(write_path.to_string()); + let mut model = src.clone(); - info!("saved: {:?}", model); + model.last_saved = Some(now); + model.written_to.insert(write_path.to_string()); - Ok(model) + Ok(model) + } } /// copy from src to dest diff --git a/src/bin/replica.rs b/src/bin/replica.rs index 3f3d14d..99f4a5d 100644 --- a/src/bin/replica.rs +++ b/src/bin/replica.rs @@ -6,9 +6,10 @@ use clap::Parser; use log::{error, info, warn}; use replica::backup_process::BackupProcess; use replica::config::Config; -use replica::file_model::FileModel; use replica::file_walker::FileWalker; +use replica::kv_store::KeyValueStore; use std::env; +use std::path::PathBuf; use std::time::Instant; #[derive(Clone, Debug, Default, Parser)] @@ -34,14 +35,19 @@ fn cd_app_home(app_home: &str) { env::set_current_dir(app_home).unwrap_or_else(|_| panic!("{}", msg)); } +fn startup(cli: &Cli) -> Config { + let config = Config::read_config(cli.config.as_str()).expect("config should initialize"); + config.start_logger().expect("logger should start."); + + info!("replica config: {:?}", config); + + config.to_owned() +} + /// the primary process -fn run(cli: Cli) -> Result<()> { +fn run(cli: Cli, config: Config) -> Result<()> { let start_time = Instant::now(); - let config = Config::read_config(cli.config.as_str())?; - - config.start_logger()?; - info!("replica config: {:?}", config); cd_app_home(config.home.as_str()); if cli.dryrun { @@ -49,6 +55,7 @@ fn run(cli: Cli) -> Result<()> { } // read the current database DbOps + let mut db = KeyValueStore::init(PathBuf::from(config.dbfile.clone()))?; let walker = FileWalker::new(config.clone()); if let Ok(files) = walker.walk_files_and_folders() { @@ -56,18 +63,16 @@ fn run(cli: Cli) -> Result<()> { let target_dir = &config.targets[0]; let backup = BackupProcess::new(target_dir.as_str(), files.clone(), cli.dryrun); - let results = backup.process(); - if results.is_ok() { - let saved_list = results.unwrap(); - let count = saved_list.len(); - info!("{} files backed up.", count); - // now update the db file records - if count > 0 { - let dbvec = FileModel::merge_updates(files, saved_list); - let _ = FileModel::write_dbfile(&config.dbfile, dbvec); - } - } else { - error!("{:?}", results); + let results = backup.process(db.clone()); + if results.is_err() { + error!("backup failed: {:?}", results); + } + } + + if db.is_dirty() { + let resp = db.savedb(config.dbfile.as_str()); + if resp.is_err() { + error!("database save failed: {:?}", resp); } } @@ -82,27 +87,73 @@ fn main() -> Result<()> { let home = env::var("HOME").expect("The user should have a home folder."); cd_app_home(home.as_str()); - run(Cli::parse()) + let cli = Cli::parse(); + let config = startup(&cli); + run(cli, config) } #[cfg(test)] mod tests { use super::*; + use std::{fs::File, io::Write}; + + fn change_file() { + let filename = "tests/changed-file.txt"; + let mut buf = File::create(filename).unwrap(); + let msg = format!("the time: {:?}", Instant::now()); + buf.write_all(msg.as_bytes()).unwrap(); + } #[test] - fn find_exepath() { - let exepath = env::current_exe().expect("should have an exepath"); + fn startup_test() { + let test_home = env::current_dir().expect("should get the current working directory"); + let conf_path = format!( + "{}/.test-replica/config/run-config.toml", + test_home.to_str().unwrap() + ); - println!("exe path: {}", exepath.display()); + let cli = Cli { + config: conf_path, + verbose: false, + dryrun: false, + }; + + let config = startup(&cli); + println!("cli: {:?}", cli); + println!("ctx: {:?}", config); + + assert!(true); } #[test] fn run_test() { + change_file(); let test_home = env::current_dir().expect("should get the current working directory"); let conf_path = format!( "{}/.test-replica/config/run-config.toml", test_home.to_str().unwrap() ); + let config = Config::read_config(conf_path.as_str()).unwrap(); + + println!("conf path : {:?}", conf_path); + let cli = Cli { + config: conf_path, + verbose: false, + dryrun: false, + }; + println!("{:?}", cli); + let results = run(cli, config); + assert!(results.is_ok()); + } + + #[test] + fn run_test_dryrun() { + let test_home = env::current_dir().expect("should get the current working directory"); + let conf_path = format!( + "{}/.test-replica/config/run-config.toml", + test_home.to_str().unwrap() + ); + let config = Config::read_config(conf_path.as_str()).unwrap(); println!("conf path : {:?}", conf_path); let cli = Cli { config: conf_path, @@ -110,7 +161,8 @@ mod tests { dryrun: true, }; println!("{:?}", cli); - let results = run(cli); + let results = run(cli, config); + println!("{:?}", results); assert!(results.is_ok()); } @@ -120,4 +172,11 @@ mod tests { println!("{}", test_home.display()); cd_app_home(test_home.to_str().unwrap()); } + + #[test] + fn find_exepath() { + let exepath = env::current_exe().expect("should have an exepath"); + + println!("exe path: {}", exepath.display()); + } } diff --git a/src/file_model.rs b/src/file_model.rs index 57753a1..ccc18bf 100644 --- a/src/file_model.rs +++ b/src/file_model.rs @@ -5,13 +5,11 @@ use anyhow::{anyhow, Result}; use chrono::naive::NaiveDateTime; use domain_keys::keys::RouteKey; -use hashbrown::{HashMap, HashSet}; -use log::{error, info, warn}; +use hashbrown::HashSet; +use log::error; use openssl::sha; use serde::{Deserialize, Serialize}; use std::env; -use std::fs::File; -use std::io::{BufReader, Read, Write}; use std::path::PathBuf; #[derive(Debug, Default, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -66,7 +64,14 @@ impl FileModel { /// read the file based metadata, len, modified, etc pub fn read_metadata(&self) -> Result { let mut model = self.clone(); - let meta = self.path.metadata()?; + let resp = self.path.metadata(); + if resp.is_err() { + let msg = format!("failed to get meta data: {:?}", resp); + error!("{}", msg); + return Err(anyhow!("{}", msg)); + } + + let meta = resp.unwrap(); model.len = meta.len(); let modified = meta.modified().unwrap(); @@ -87,46 +92,6 @@ impl FileModel { hex::encode(hash) } - /// read the db file - pub fn read_dbfile(filename: &str) -> Result> { - // check to see if the file exists... - info!("read db model list: {}", filename); - let file = match File::open(filename) { - Ok(file) => file, - Err(e) => { - warn!("creating a new empty list: {}", e); - let list: Vec = Vec::new(); - return Ok(list); - } - }; - - let mut reader = BufReader::new(file); - - let mut text = String::new(); - reader.read_to_string(&mut text)?; - - let list: Vec = serde_json::from_str(&text)?; - - Ok(list) - } - - /// save the list of file models to disk - pub fn write_dbfile(filename: &str, list: Vec) -> Result<()> { - info!("write model list to file: {}", filename); - let json = serde_json::to_string_pretty(&list).unwrap(); - - match File::create(filename) { - Ok(mut buf) => buf.write_all(json.as_bytes())?, - Err(e) => { - let msg = format!("dbfile write error: {}, {}", filename, e); - error!("{}", msg); - return Err(anyhow!("{}", msg)); - } - } - - Ok(()) - } - /// strip off the home parts to return the relative path pub fn relative_path(&self) -> String { let mut home = env::var("HOME").expect("The user should have a home folder."); @@ -135,74 +100,12 @@ impl FileModel { } self.path.to_str().unwrap().replace(home.as_str(), "") } - - /// update the reference files with saved list - pub fn merge_updates(files: Vec, saved: Vec) -> Vec { - info!( - "update the reference with saved files, ref: {}, saved: {}", - files.len(), - saved.len() - ); - - let mut hmap: HashMap = - files.into_iter().map(|v| (v.key.to_string(), v)).collect(); - - for model in saved { - let key = model.key.clone(); - hmap.insert(key, model); - } - - hmap.into_values().collect() - } } #[cfg(test)] mod tests { use super::*; - #[test] - fn merge_updates() { - let filename = "tests/data/files.json"; - let ref_list = FileModel::read_dbfile(filename).expect("a vector of file models"); - assert_eq!(ref_list.len(), 5); - let filename = "tests/data/saved.json"; - let saved = FileModel::read_dbfile(filename).expect("a vector of file models"); - assert_eq!(saved.len(), 3); - - println!("{:?}", saved); - - let list = FileModel::merge_updates(ref_list.clone(), saved); - assert_eq!(ref_list.len(), list.len()); - } - - #[test] - fn write_file_bad() { - let filename = "tests/data/files.json"; - let list = FileModel::read_dbfile(filename).expect("a vector of file models"); - - assert_eq!(list.len(), 5); - let filename = "bad-path/nothere/bad-file-name"; - let result = FileModel::write_dbfile(filename, list); - println!("err: {:?}", result); - assert!(result.is_err()); - } - - #[test] - fn read_dbfile() { - let filename = "tests/data/files.json"; - let list = FileModel::read_dbfile(filename).expect("a vector of file models"); - - assert_eq!(list.len(), 5); - } - - #[test] - fn read_dbfile_bad() { - let filename = ".test-replica/data/no-files.json"; - let list = FileModel::read_dbfile(filename).expect("a vector of file models"); - - assert_eq!(list.len(), 0); - } - #[test] fn calc_hash() { let model = FileModel::new("config/config.toml"); diff --git a/src/kv_store.rs b/src/kv_store.rs index 28bfc26..59cf657 100644 --- a/src/kv_store.rs +++ b/src/kv_store.rs @@ -80,7 +80,7 @@ impl KeyValueStore { } /// return the size of this database - pub fn len(&self) -> usize { + pub fn dbsize(&self) -> usize { self.db.len() } @@ -92,9 +92,7 @@ impl KeyValueStore { /// find the file model from the path pub fn find(&self, path: &str) -> Option<&FileModel> { let key = self.index.get(path); - if key.is_none() { - return None; - } + key?; self.db.get(key.unwrap()) } @@ -125,6 +123,16 @@ impl KeyValueStore { mod tests { use super::*; + #[test] + fn savedb_bad() { + let filename = "tests/data/files.json"; + let mut client = KeyValueStore::init(PathBuf::from(filename)).unwrap(); + + let filename = "tests/no-data/bad/backup.json"; + let result = client.savedb(filename); + assert!(result.is_err()) + } + #[test] fn savedb() { let filename = "tests/data/files.json"; @@ -150,12 +158,12 @@ mod tests { } #[test] - fn getset_dirty_len() { + fn getset_dirty_dbsize() { let filename = "tests/data/files.json"; let mut client = KeyValueStore::init(PathBuf::from(filename)).unwrap(); assert!(!client.is_dirty()); let count: usize = 5; - assert_eq!(client.len(), count); + assert_eq!(client.dbsize(), count); let result = client.get("bad-key"); assert!(result.is_none()); @@ -179,7 +187,7 @@ mod tests { let updated = client.get(&model.key).unwrap(); println!("up: {:?}", updated); assert_eq!(updated.hash, myhash); - assert_eq!(client.len(), count); + assert_eq!(client.dbsize(), count); } #[test] diff --git a/tests/changed-file.txt b/tests/changed-file.txt new file mode 100644 index 0000000..1584885 --- /dev/null +++ b/tests/changed-file.txt @@ -0,0 +1 @@ +the time: Instant { tv_sec: 3756506, tv_nsec: 796245450 } \ No newline at end of file From 5c056d53ee3075f1a5ce3cb46607d780a7f5c5f4 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 17:04:00 -0800 Subject: [PATCH 12/13] updated to fix reference bug --- .test-replica/data/run2-file.json | 280 +++++++++++++++++++++++++++++- src/backup_process.rs | 4 +- src/bin/replica.rs | 15 +- tests/changed-file.txt | 2 +- 4 files changed, 283 insertions(+), 18 deletions(-) diff --git a/.test-replica/data/run2-file.json b/.test-replica/data/run2-file.json index 3d8968d..3d5ae1b 100644 --- a/.test-replica/data/run2-file.json +++ b/.test-replica/data/run2-file.json @@ -1,7 +1,183 @@ [ { - "key": "gfl27msaerQYZ1Vy", - "path": "tback/./tests/.config/file1.txt", + "key": "gItM7mskxeejVoGs", + "path": "./tests/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:16.517783992", + "written_to": [ + "tback/./tests/file1.txt" + ] + }, + { + "key": "Norm7msl0dQLwUQi", + "path": "./tests/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:00.553220967", + "written_to": [ + "tback/./tests/file2.txt" + ] + }, + { + "key": "zPPZ7msl38fVOsu3", + "path": "./tests/.config/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:37.556572254", + "written_to": [ + "tback/./tests/.config/file1.txt" + ] + }, + { + "key": "ytEw7msl38f4DHJr", + "path": "./tests/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:37.555494554", + "written_to": [ + "tback/./tests/file1.txt" + ] + }, + { + "key": "BP3v7msl38fDdFDG", + "path": "./tests/big-file.pdf", + "hash": "", + "len": 474707, + "modified": 1699990650530503, + "last_saved": "2023-11-21T01:02:37.556382986", + "written_to": [ + "tback/./tests/big-file.pdf" + ] + }, + { + "key": "xd7M7mskxKz8QSId", + "path": "./tests/.config/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:11.830740480", + "written_to": [ + "tback/./tests/.config/file1.txt" + ] + }, + { + "key": "zPKK7msl0dQLitM5", + "path": "./tests/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:00.553172599", + "written_to": [ + "tback/./tests/file1.txt" + ] + }, + { + "key": "BMs47mskxeejlTj4", + "path": "./tests/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:16.517986371", + "written_to": [ + "tback/./tests/file2.txt" + ] + }, + { + "key": "YyLl7mskxeejrMRu", + "path": "./tests/big-file.pdf", + "hash": "", + "len": 474707, + "modified": 1699990650530503, + "last_saved": "2023-11-21T01:01:16.518476510", + "written_to": [ + "tback/./tests/big-file.pdf" + ] + }, + { + "key": "cm3k7mskxeekb3IY", + "path": "./tests/.config/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:16.518801601", + "written_to": [ + "tback/./tests/.config/file2.txt" + ] + }, + { + "key": "pNjN7msl0dQMmU6K", + "path": "./tests/.config/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:00.553991478", + "written_to": [ + "tback/./tests/.config/file2.txt" + ] + }, + { + "key": "LOqI7msl0dQMfjcK", + "path": "./tests/.config/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:00.553900103", + "written_to": [ + "tback/./tests/.config/file1.txt" + ] + }, + { + "key": "ccwK7mskxKz70F0S", + "path": "./tests/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:11.829766865", + "written_to": [ + "tback/./tests/file1.txt" + ] + }, + { + "key": "r4L47msl38fXGYZL", + "path": "./tests/.config/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:37.556652023", + "written_to": [ + "tback/./tests/.config/file2.txt" + ] + }, + { + "key": "1ok47msl38fDXbTi", + "path": "./tests/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:02:37.556051416", + "written_to": [ + "tback/./tests/file2.txt" + ] + }, + { + "key": "QSdQ7mskxKz7LsHS", + "path": "./tests/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:11.829913781", + "written_to": [ + "tback/./tests/file2.txt" + ] + }, + { + "key": "9x2h7msaerQZU733", + "path": "tback/./tests/.config/file2.txt", "hash": "", "len": 0, "modified": 0, @@ -9,14 +185,36 @@ "written_to": [] }, { - "key": "9x2h7msaerQZU733", - "path": "tback/./tests/.config/file2.txt", + "key": "ZuGm7msl0dQM947R", + "path": "./tests/changed-file.txt", + "hash": "", + "len": 57, + "modified": 1700528520545191, + "last_saved": "2023-11-21T01:02:00.553775784", + "written_to": [ + "tback/./tests/changed-file.txt" + ] + }, + { + "key": "gfl27msaerQYZ1Vy", + "path": "tback/./tests/.config/file1.txt", "hash": "", "len": 0, "modified": 0, "last_saved": null, "written_to": [] }, + { + "key": "Kb6t7mskxKz7WPTc", + "path": "./tests/big-file.pdf", + "hash": "", + "len": 474707, + "modified": 1699990650530503, + "last_saved": "2023-11-21T01:01:11.830497053", + "written_to": [ + "tback/./tests/big-file.pdf" + ] + }, { "key": "p4qg7msaerQPepHa", "path": "tback/./tests/big-file.pdf", @@ -27,8 +225,52 @@ "written_to": [] }, { - "key": "XuZ57msaerQLJtwC", - "path": "tback/./tests/file1.txt", + "key": "f5l67mskxKz8c6vB", + "path": "./tests/.config/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:11.830828228", + "written_to": [ + "tback/./tests/.config/file2.txt" + ] + }, + { + "key": "Gr3E7msl0dQM37S3", + "path": "./tests/big-file.pdf", + "hash": "", + "len": 474707, + "modified": 1699990650530503, + "last_saved": "2023-11-21T01:02:00.553653208", + "written_to": [ + "tback/./tests/big-file.pdf" + ] + }, + { + "key": "9bNB7mskxeekUrj5", + "path": "./tests/.config/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T01:01:16.518712049", + "written_to": [ + "tback/./tests/.config/file1.txt" + ] + }, + { + "key": "1jiG7msl38fDiLRx", + "path": "./tests/changed-file.txt", + "hash": "", + "len": 57, + "modified": 1700528557533501, + "last_saved": "2023-11-21T01:02:37.556464479", + "written_to": [ + "tback/./tests/changed-file.txt" + ] + }, + { + "key": "sSzd7msaerQPXcfy", + "path": "tback/./tests/file2.txt", "hash": "", "len": 0, "modified": 0, @@ -36,8 +278,30 @@ "written_to": [] }, { - "key": "sSzd7msaerQPXcfy", - "path": "tback/./tests/file2.txt", + "key": "GJYg7mskxeejxMHM", + "path": "./tests/changed-file.txt", + "hash": "", + "len": 57, + "modified": 1700528476516822, + "last_saved": "2023-11-21T01:01:16.518573400", + "written_to": [ + "tback/./tests/changed-file.txt" + ] + }, + { + "key": "UmtZ7mskxKz7hY2h", + "path": "./tests/changed-file.txt", + "hash": "", + "len": 56, + "modified": 1700528471824783, + "last_saved": "2023-11-21T01:01:11.830618942", + "written_to": [ + "tback/./tests/changed-file.txt" + ] + }, + { + "key": "XuZ57msaerQLJtwC", + "path": "tback/./tests/file1.txt", "hash": "", "len": 0, "modified": 0, diff --git a/src/backup_process.rs b/src/backup_process.rs index 4367959..e45062e 100644 --- a/src/backup_process.rs +++ b/src/backup_process.rs @@ -35,7 +35,7 @@ impl BackupProcess { } /// process the file list; return the list of files that were backup - pub fn process(&self, mut db: KeyValueStore) -> Result<()> { + pub fn process(&self, mut db: KeyValueStore) -> Result { info!("process the backup queue"); let files = self.files.clone(); @@ -57,7 +57,7 @@ impl BackupProcess { } } - Ok(()) + Ok(db) } /// create the target path; check stat to see backup is required diff --git a/src/bin/replica.rs b/src/bin/replica.rs index 99f4a5d..2488d0b 100644 --- a/src/bin/replica.rs +++ b/src/bin/replica.rs @@ -66,13 +66,14 @@ fn run(cli: Cli, config: Config) -> Result<()> { let results = backup.process(db.clone()); if results.is_err() { error!("backup failed: {:?}", results); - } - } - - if db.is_dirty() { - let resp = db.savedb(config.dbfile.as_str()); - if resp.is_err() { - error!("database save failed: {:?}", resp); + } else { + db = results.unwrap(); + if db.is_dirty() { + let resp = db.savedb(config.dbfile.as_str()); + if resp.is_err() { + error!("database save failed: {:?}", resp); + } + } } } diff --git a/tests/changed-file.txt b/tests/changed-file.txt index 1584885..9375774 100644 --- a/tests/changed-file.txt +++ b/tests/changed-file.txt @@ -1 +1 @@ -the time: Instant { tv_sec: 3756506, tv_nsec: 796245450 } \ No newline at end of file +the time: Instant { tv_sec: 3757088, tv_nsec: 768845687 } \ No newline at end of file From be7861de76bb55ced4c81ca71ae79cd789633db3 Mon Sep 17 00:00:00 2001 From: darryl west Date: Mon, 20 Nov 2023 18:51:39 -0800 Subject: [PATCH 13/13] updates --- .test-replica/data/run2-file.json | 136 +++++++++++++++++++++++++++++- Cargo.lock | 2 +- Cargo.toml | 2 +- src/kv_store.rs | 2 + tests/changed-file.txt | 2 +- 5 files changed, 139 insertions(+), 5 deletions(-) diff --git a/.test-replica/data/run2-file.json b/.test-replica/data/run2-file.json index 3d5ae1b..a30edea 100644 --- a/.test-replica/data/run2-file.json +++ b/.test-replica/data/run2-file.json @@ -44,12 +44,23 @@ ] }, { - "key": "BP3v7msl38fDdFDG", + "key": "uA6q7msouXoFbf1H", + "path": "./tests/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:01:35.088522377", + "written_to": [ + "tback/./tests/file1.txt" + ] + }, + { + "key": "H7fh7msouXoFt493", "path": "./tests/big-file.pdf", "hash": "", "len": 474707, "modified": 1699990650530503, - "last_saved": "2023-11-21T01:02:37.556382986", + "last_saved": "2023-11-21T02:01:35.089120465", "written_to": [ "tback/./tests/big-file.pdf" ] @@ -65,6 +76,17 @@ "tback/./tests/.config/file1.txt" ] }, + { + "key": "BP3v7msl38fDdFDG", + "path": "./tests/big-file.pdf", + "hash": "", + "len": 474707, + "modified": 1699990650530503, + "last_saved": "2023-11-21T01:02:37.556382986", + "written_to": [ + "tback/./tests/big-file.pdf" + ] + }, { "key": "zPKK7msl0dQLitM5", "path": "./tests/file1.txt", @@ -109,6 +131,28 @@ "tback/./tests/.config/file2.txt" ] }, + { + "key": "0gBH7msouXoFnkzK", + "path": "./tests/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:01:35.088654972", + "written_to": [ + "tback/./tests/file2.txt" + ] + }, + { + "key": "0NE97mspHtaliAP1", + "path": "./tests/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:07:20.135639811", + "written_to": [ + "tback/./tests/file1.txt" + ] + }, { "key": "pNjN7msl0dQMmU6K", "path": "./tests/.config/file2.txt", @@ -120,6 +164,17 @@ "tback/./tests/.config/file2.txt" ] }, + { + "key": "06gK7mspHtam6vC0", + "path": "./tests/changed-file.txt", + "hash": "", + "len": 57, + "modified": 1700532440130046, + "last_saved": "2023-11-21T02:07:20.136257746", + "written_to": [ + "tback/./tests/changed-file.txt" + ] + }, { "key": "LOqI7msl0dQMfjcK", "path": "./tests/.config/file1.txt", @@ -131,6 +186,28 @@ "tback/./tests/.config/file1.txt" ] }, + { + "key": "qKhx7mspHtamakty", + "path": "./tests/.config/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:07:20.136379799", + "written_to": [ + "tback/./tests/.config/file1.txt" + ] + }, + { + "key": "bSZJ7msouXoFzgVd", + "path": "./tests/changed-file.txt", + "hash": "", + "len": 57, + "modified": 1700532095083149, + "last_saved": "2023-11-21T02:01:35.089227356", + "written_to": [ + "tback/./tests/changed-file.txt" + ] + }, { "key": "ccwK7mskxKz70F0S", "path": "./tests/file1.txt", @@ -164,6 +241,17 @@ "tback/./tests/file2.txt" ] }, + { + "key": "MCK47mspHtamiVP7", + "path": "./tests/.config/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:07:20.136466845", + "written_to": [ + "tback/./tests/.config/file2.txt" + ] + }, { "key": "QSdQ7mskxKz7LsHS", "path": "./tests/file2.txt", @@ -257,6 +345,17 @@ "tback/./tests/.config/file1.txt" ] }, + { + "key": "a2tw7mspHtaluIPy", + "path": "./tests/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:07:20.135749002", + "written_to": [ + "tback/./tests/file2.txt" + ] + }, { "key": "1jiG7msl38fDiLRx", "path": "./tests/changed-file.txt", @@ -277,6 +376,17 @@ "last_saved": null, "written_to": [] }, + { + "key": "45eV7msouXoGTnuY", + "path": "./tests/.config/file1.txt", + "hash": "", + "len": 186, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:01:35.089400847", + "written_to": [ + "tback/./tests/.config/file1.txt" + ] + }, { "key": "GJYg7mskxeejxMHM", "path": "./tests/changed-file.txt", @@ -299,6 +409,28 @@ "tback/./tests/changed-file.txt" ] }, + { + "key": "JBIS7msouXoGZ0lI", + "path": "./tests/.config/file2.txt", + "hash": "", + "len": 19, + "modified": 1700331937339576, + "last_saved": "2023-11-21T02:01:35.089504300", + "written_to": [ + "tback/./tests/.config/file2.txt" + ] + }, + { + "key": "ZuvO7mspHtam01v9", + "path": "./tests/big-file.pdf", + "hash": "", + "len": 474707, + "modified": 1699990650530503, + "last_saved": "2023-11-21T02:07:20.136161875", + "written_to": [ + "tback/./tests/big-file.pdf" + ] + }, { "key": "XuZ57msaerQLJtwC", "path": "tback/./tests/file1.txt", diff --git a/Cargo.lock b/Cargo.lock index bcadefc..4a546bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -885,7 +885,7 @@ dependencies = [ [[package]] name = "replica" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 95cffc2..6994df7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "replica" -version = "0.4.1" +version = "0.4.2" edition = "2021" authors = ["darryl.west@raincitysoftware.com"] rust-version = "1.70" diff --git a/src/kv_store.rs b/src/kv_store.rs index 59cf657..2feb577 100644 --- a/src/kv_store.rs +++ b/src/kv_store.rs @@ -100,6 +100,8 @@ impl KeyValueStore { /// save the kv to file pub fn savedb(&mut self, filename: &str) -> Result<()> { info!("save the k/v models as a list to file: {}", filename); + // ok, first copy the current to a time-stamped replica. + let list: Vec = self.db.clone().into_values().collect(); let json = serde_json::to_string_pretty(&list).unwrap(); diff --git a/tests/changed-file.txt b/tests/changed-file.txt index 9375774..28f46ae 100644 --- a/tests/changed-file.txt +++ b/tests/changed-file.txt @@ -1 +1 @@ -the time: Instant { tv_sec: 3757088, tv_nsec: 768845687 } \ No newline at end of file +the time: Instant { tv_sec: 3760971, tv_nsec: 365143245 } \ No newline at end of file