From 8490c031887dd5fd14b47dc37457de59d2f7330e Mon Sep 17 00:00:00 2001 From: Tarek Date: Tue, 12 Mar 2024 05:44:30 +0200 Subject: [PATCH 1/4] factor a crate fs-metadata out of fs-storage Signed-off-by: Tarek --- Cargo.toml | 3 +++ README.md | 1 + data-link/Cargo.toml | 1 + data-link/src/lib.rs | 2 +- fs-metadata/Cargo.toml | 17 +++++++++++++++++ .../src/meta.rs => fs-metadata/src/lib.rs | 4 +++- fs-storage/src/lib.rs | 2 -- 7 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 fs-metadata/Cargo.toml rename fs-storage/src/meta.rs => fs-metadata/src/lib.rs (96%) diff --git a/Cargo.toml b/Cargo.toml index 1a75dffd..54efc50e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "data-resource", "fs-atomic-versions", "fs-atomic-light", + "fs-metadata", "fs-index", "fs-storage", ] @@ -18,6 +19,8 @@ default-members = [ "data-pdf", "data-resource", "fs-atomic-versions", + "fs-atomic-light", + "fs-metadata", "fs-index", "fs-storage", ] diff --git a/README.md b/README.md index 1f6cadbd..ac9c271f 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ The purpose of the library is to manage _resource index_ of folders with various | `data-resource` | Resource hashing and ID construction | | `fs-index` | Resource Index construction and updating | | `fs-storage` | Filesystem storage for resources | +| `fs-metadata` | Metadata management | | `data-link` | Linking resources | | `data-pdf` | PDF handling | | `data-error` | Error handling | diff --git a/data-link/Cargo.toml b/data-link/Cargo.toml index 10a8c5bc..e6e20aba 100644 --- a/data-link/Cargo.toml +++ b/data-link/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["rlib"] fs-atomic-light = { path = "../fs-atomic-light" } fs-atomic-versions = { path = "../fs-atomic-versions" } fs-storage = { path = "../fs-storage" } +fs-metadata = { path = "../fs-metadata" } data-resource = { path = "../data-resource" } data-error = { path = "../data-error" } diff --git a/data-link/src/lib.rs b/data-link/src/lib.rs index b02391d8..e4faa915 100644 --- a/data-link/src/lib.rs +++ b/data-link/src/lib.rs @@ -1,7 +1,7 @@ use data_error::Result; use data_resource::ResourceId; use fs_atomic_versions::atomic::AtomicFile; -use fs_storage::meta::store_metadata; +use fs_metadata::store_metadata; use fs_storage::prop::load_raw_properties; use fs_storage::prop::store_properties; use fs_storage::{ diff --git a/fs-metadata/Cargo.toml b/fs-metadata/Cargo.toml new file mode 100644 index 00000000..184a341f --- /dev/null +++ b/fs-metadata/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "fs-metadata" +version = "0.1.0" +edition = "2021" + + +[dependencies] +data-error = { path = "../data-error" } +fs-atomic-versions = { path = "../fs-atomic-versions" } +data-resource = { path = "../data-resource" } +fs-storage = { path = "../fs-storage" } +serde = { version = "1.0.138", features = ["derive"] } + +[dev-dependencies] +log = { version = "0.4.17", features = ["release_max_level_off"] } +tempdir = "0.3.7" +serde_json = "1.0.82" diff --git a/fs-storage/src/meta.rs b/fs-metadata/src/lib.rs similarity index 96% rename from fs-storage/src/meta.rs rename to fs-metadata/src/lib.rs index 5ada9b72..3118fd4d 100644 --- a/fs-storage/src/meta.rs +++ b/fs-metadata/src/lib.rs @@ -5,8 +5,10 @@ use std::fmt::Debug; use std::io::Read; use std::path::Path; -use crate::{ARK_FOLDER, METADATA_STORAGE_FOLDER}; use data_resource::ResourceId; +use fs_storage::ARK_FOLDER; + +pub const METADATA_STORAGE_FOLDER: &str = "cache/metadata"; pub fn store_metadata< S: Serialize + DeserializeOwned + Clone + Debug, diff --git a/fs-storage/src/lib.rs b/fs-storage/src/lib.rs index 90cb61a5..8cc22f23 100644 --- a/fs-storage/src/lib.rs +++ b/fs-storage/src/lib.rs @@ -1,4 +1,3 @@ -pub mod meta; pub mod prop; pub const ARK_FOLDER: &str = ".ark"; @@ -14,6 +13,5 @@ pub const PROPERTIES_STORAGE_FOLDER: &str = "user/properties"; // Generated data pub const INDEX_PATH: &str = "index"; -pub const METADATA_STORAGE_FOLDER: &str = "cache/metadata"; pub const PREVIEWS_STORAGE_FOLDER: &str = "cache/previews"; pub const THUMBNAILS_STORAGE_FOLDER: &str = "cache/thumbnails"; From 43ba24da0afdf20280bf4bf9ff1654c814bd5769 Mon Sep 17 00:00:00 2001 From: Tarek Date: Tue, 12 Mar 2024 05:51:51 +0200 Subject: [PATCH 2/4] define a crate fs-properties to manage properties Signed-off-by: Tarek --- Cargo.toml | 2 ++ README.md | 1 + data-link/Cargo.toml | 1 + data-link/src/lib.rs | 9 ++++----- fs-properties/Cargo.toml | 19 +++++++++++++++++++ .../src/prop.rs => fs-properties/src/lib.rs | 4 +++- fs-storage/Cargo.toml | 11 ----------- fs-storage/src/lib.rs | 3 --- 8 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 fs-properties/Cargo.toml rename fs-storage/src/prop.rs => fs-properties/src/lib.rs (96%) diff --git a/Cargo.toml b/Cargo.toml index 54efc50e..baba308d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "fs-atomic-versions", "fs-atomic-light", "fs-metadata", + "fs-properties", "fs-index", "fs-storage", ] @@ -21,6 +22,7 @@ default-members = [ "fs-atomic-versions", "fs-atomic-light", "fs-metadata", + "fs-properties", "fs-index", "fs-storage", ] diff --git a/README.md b/README.md index ac9c271f..9f7b003e 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The purpose of the library is to manage _resource index_ of folders with various | `fs-index` | Resource Index construction and updating | | `fs-storage` | Filesystem storage for resources | | `fs-metadata` | Metadata management | +| `fs-properties` | Properties management | | `data-link` | Linking resources | | `data-pdf` | PDF handling | | `data-error` | Error handling | diff --git a/data-link/Cargo.toml b/data-link/Cargo.toml index e6e20aba..3fe9642b 100644 --- a/data-link/Cargo.toml +++ b/data-link/Cargo.toml @@ -12,6 +12,7 @@ fs-atomic-light = { path = "../fs-atomic-light" } fs-atomic-versions = { path = "../fs-atomic-versions" } fs-storage = { path = "../fs-storage" } fs-metadata = { path = "../fs-metadata" } +fs-properties = { path = "../fs-properties" } data-resource = { path = "../data-resource" } data-error = { path = "../data-error" } diff --git a/data-link/src/lib.rs b/data-link/src/lib.rs index e4faa915..267d0de3 100644 --- a/data-link/src/lib.rs +++ b/data-link/src/lib.rs @@ -2,11 +2,10 @@ use data_error::Result; use data_resource::ResourceId; use fs_atomic_versions::atomic::AtomicFile; use fs_metadata::store_metadata; -use fs_storage::prop::load_raw_properties; -use fs_storage::prop::store_properties; -use fs_storage::{ - ARK_FOLDER, PREVIEWS_STORAGE_FOLDER, PROPERTIES_STORAGE_FOLDER, -}; +use fs_properties::load_raw_properties; +use fs_properties::store_properties; +use fs_properties::PROPERTIES_STORAGE_FOLDER; +use fs_storage::{ARK_FOLDER, PREVIEWS_STORAGE_FOLDER}; use reqwest::header::HeaderValue; use scraper::{Html, Selector}; use serde::{Deserialize, Serialize}; diff --git a/fs-properties/Cargo.toml b/fs-properties/Cargo.toml new file mode 100644 index 00000000..6ce599a8 --- /dev/null +++ b/fs-properties/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "fs-properties" +version = "0.1.0" +edition = "2021" + + +[dependencies] +data-error = { path = "../data-error" } +data-json = { path = "../data-json" } +fs-atomic-versions = { path = "../fs-atomic-versions" } +fs-storage = { path = "../fs-storage" } +data-resource = { path = "../data-resource" } + +serde_json = "1.0.82" +serde = { version = "1.0.138", features = ["derive"] } + +[dev-dependencies] +log = { version = "0.4.17", features = ["release_max_level_off"] } +tempdir = "0.3.7" diff --git a/fs-storage/src/prop.rs b/fs-properties/src/lib.rs similarity index 96% rename from fs-storage/src/prop.rs rename to fs-properties/src/lib.rs index e09cd2c9..45a71b9e 100644 --- a/fs-storage/src/prop.rs +++ b/fs-properties/src/lib.rs @@ -7,8 +7,10 @@ use std::fmt::Debug; use std::io::Read; use std::path::Path; -use crate::{ARK_FOLDER, PROPERTIES_STORAGE_FOLDER}; use data_resource::ResourceId; +use fs_storage::ARK_FOLDER; + +pub const PROPERTIES_STORAGE_FOLDER: &str = "user/properties"; pub fn store_properties< S: Serialize + DeserializeOwned + Clone + Debug, diff --git a/fs-storage/Cargo.toml b/fs-storage/Cargo.toml index 146e6c98..93bd7038 100644 --- a/fs-storage/Cargo.toml +++ b/fs-storage/Cargo.toml @@ -5,14 +5,3 @@ edition = "2021" [dependencies] -data-error = { path = "../data-error" } -data-json = { path = "../data-json" } -fs-atomic-versions = { path = "../fs-atomic-versions" } -data-resource = { path = "../data-resource" } - -serde_json = "1.0.82" -serde = { version = "1.0.138", features = ["derive"] } - -[dev-dependencies] -log = { version = "0.4.17", features = ["release_max_level_off"] } -tempdir = "0.3.7" diff --git a/fs-storage/src/lib.rs b/fs-storage/src/lib.rs index 8cc22f23..6e7af127 100644 --- a/fs-storage/src/lib.rs +++ b/fs-storage/src/lib.rs @@ -1,5 +1,3 @@ -pub mod prop; - pub const ARK_FOLDER: &str = ".ark"; // Should not be lost if possible @@ -9,7 +7,6 @@ pub const FAVORITES_FILE: &str = "favorites"; // User-defined data pub const TAG_STORAGE_FILE: &str = "user/tags"; pub const SCORE_STORAGE_FILE: &str = "user/scores"; -pub const PROPERTIES_STORAGE_FOLDER: &str = "user/properties"; // Generated data pub const INDEX_PATH: &str = "index"; From ba41bd8690c03920cfaaf0d0f94c13a7342e950d Mon Sep 17 00:00:00 2001 From: Tarek Date: Wed, 13 Mar 2024 03:05:16 +0200 Subject: [PATCH 3/4] fix: pin image crate to version 0.25 Signed-off-by: Tarek --- data-pdf/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-pdf/Cargo.toml b/data-pdf/Cargo.toml index a1131e1d..497c8bc3 100644 --- a/data-pdf/Cargo.toml +++ b/data-pdf/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["rlib"] [dependencies] once_cell = "1.16.0" -image = "0.24.2" +image = "=0.25.0" pdfium-render = { git = "https://github.com/ajrcarey/pdfium-render", rev = "d2559c1", features = [ "thread_safe", "sync", From 594d837ef38a5c82501a376c9744b4f9354fd204 Mon Sep 17 00:00:00 2001 From: Tarek Date: Thu, 14 Mar 2024 14:45:49 +0200 Subject: [PATCH 4/4] fix(ci): disable benchmarks for all crates in the workspace The bench feature has been disabled for all crates within the workspace to resolve an issue encountered during CI testing. The reason is that by default, Cargo implicitly adds a libtest benchmark harness to any crate when benchmarking, to handle any #[bench] functions, even if we have none. This ruins our Benchmarking CI with the error (Unrecognized option: save-baseline) in the logs. For more details: https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options Signed-off-by: Tarek --- data-error/Cargo.toml | 5 +++++ data-json/Cargo.toml | 4 ++++ data-link/Cargo.toml | 1 + data-pdf/Cargo.toml | 1 + fs-atomic-light/Cargo.toml | 1 + fs-atomic-versions/Cargo.toml | 1 + fs-metadata/Cargo.toml | 4 ++++ fs-properties/Cargo.toml | 4 ++++ fs-storage/Cargo.toml | 4 ++++ 9 files changed, 25 insertions(+) diff --git a/data-error/Cargo.toml b/data-error/Cargo.toml index b30cb2a6..63ac295d 100644 --- a/data-error/Cargo.toml +++ b/data-error/Cargo.toml @@ -3,6 +3,11 @@ name = "data-error" version = "0.1.0" edition = "2021" +[lib] +name = "data_error" +crate-type = ["rlib"] +bench = false + [dependencies] thiserror = "1" reqwest = "0.11.11" diff --git a/data-json/Cargo.toml b/data-json/Cargo.toml index b56702df..0bd7f9aa 100644 --- a/data-json/Cargo.toml +++ b/data-json/Cargo.toml @@ -3,6 +3,10 @@ name = "data-json" version = "0.1.0" edition = "2021" +[lib] +name = "data_json" +crate-type = ["rlib"] +bench = false [dependencies] serde_json = "1.0.82" diff --git a/data-link/Cargo.toml b/data-link/Cargo.toml index 3fe9642b..71c3d960 100644 --- a/data-link/Cargo.toml +++ b/data-link/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [lib] name = "data_link" crate-type = ["rlib"] +bench = false [dependencies] fs-atomic-light = { path = "../fs-atomic-light" } diff --git a/data-pdf/Cargo.toml b/data-pdf/Cargo.toml index 497c8bc3..e6f31cd5 100644 --- a/data-pdf/Cargo.toml +++ b/data-pdf/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [lib] name = "data_pdf" crate-type = ["rlib"] +bench = false [dependencies] once_cell = "1.16.0" diff --git a/fs-atomic-light/Cargo.toml b/fs-atomic-light/Cargo.toml index abb4a81c..5c15bd54 100644 --- a/fs-atomic-light/Cargo.toml +++ b/fs-atomic-light/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [lib] name = "fs_atomic_light" crate-type = ["rlib"] +bench = false [dependencies] data-error = { path = "../data-error" } diff --git a/fs-atomic-versions/Cargo.toml b/fs-atomic-versions/Cargo.toml index d3cab8e5..70372063 100644 --- a/fs-atomic-versions/Cargo.toml +++ b/fs-atomic-versions/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [lib] name = "fs_atomic_versions" crate-type = ["rlib"] +bench = false [dependencies] data-error = { path = "../data-error" } diff --git a/fs-metadata/Cargo.toml b/fs-metadata/Cargo.toml index 184a341f..89a9e03b 100644 --- a/fs-metadata/Cargo.toml +++ b/fs-metadata/Cargo.toml @@ -3,6 +3,10 @@ name = "fs-metadata" version = "0.1.0" edition = "2021" +[lib] +name = "fs_metadata" +crate-type = ["rlib"] +bench = false [dependencies] data-error = { path = "../data-error" } diff --git a/fs-properties/Cargo.toml b/fs-properties/Cargo.toml index 6ce599a8..ece81dfa 100644 --- a/fs-properties/Cargo.toml +++ b/fs-properties/Cargo.toml @@ -3,6 +3,10 @@ name = "fs-properties" version = "0.1.0" edition = "2021" +[lib] +name = "fs_properties" +crate-type = ["rlib"] +bench = false [dependencies] data-error = { path = "../data-error" } diff --git a/fs-storage/Cargo.toml b/fs-storage/Cargo.toml index 93bd7038..09fc9687 100644 --- a/fs-storage/Cargo.toml +++ b/fs-storage/Cargo.toml @@ -3,5 +3,9 @@ name = "fs-storage" version = "0.1.0" edition = "2021" +[lib] +name = "fs_storage" +crate-type = ["rlib"] +bench = false [dependencies]