Refactor ResourceIndex and Associated Methods

.github/workflows/build.yml

@@ -22,6 +22,7 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
+          toolchain: nightly # nightly is required for fmt
           components: rustfmt, clippy
 
       - name: Check

.gitignore

@@ -2,4 +2,5 @@
 target
 Cargo.lock
 **/app_id
-*.class
+*.class
+**/.ark

README.md

@@ -76,7 +76,7 @@ cargo bench
 This command runs all benchmarks and generates a report in HTML format located at `target/criterion/report`. If you wish to run a specific benchmark, you can specify its name as an argument as in:
 
 ```bash
-cargo bench index_build
+cargo bench resource_index
 ```
 
 ### Benchmarking Local Files
@@ -97,10 +97,10 @@ To install `flamegraph`, run:
 cargo install flamegraph
 ```
 
-To generate a flame graph for `index_build_benchmark`, use the following command:
+To generate a flame graph for `resource_index_benchmark`, use the following command:
 
 ```bash
-cargo flamegraph --bench index_build_benchmark -o index_build_benchmark.svg -- --bench
+cargo flamegraph --bench resource_index_benchmark -o resource_index_benchmark.svg -- --bench
 ```
 
 > [!NOTE]

ark-cli/src/commands/backup.rs

@@ -1,5 +1,4 @@
-use std::io::Write;
-use std::path::PathBuf;
+use std::{io::Write, path::PathBuf};
 
 use crate::{
     create_dir_all, dir, discover_roots, home_dir, storages_exists, timestamp,

ark-cli/src/commands/file/append.rs

@@ -1,9 +1,8 @@
-use std::path::PathBuf;
-use std::str::FromStr;
+use std::{path::PathBuf, str::FromStr};
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError, Format, ResourceId,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError, Format, ResourceId,
 };
 
 use data_error::ArklibError;

ark-cli/src/commands/file/insert.rs

@@ -1,9 +1,8 @@
-use std::path::PathBuf;
-use std::str::FromStr;
+use std::{path::PathBuf, str::FromStr};
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError, Format, ResourceId,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError, Format, ResourceId,
 };
 
 use data_error::ArklibError;

ark-cli/src/commands/file/read.rs

@@ -1,9 +1,8 @@
-use std::path::PathBuf;
-use std::str::FromStr;
+use std::{path::PathBuf, str::FromStr};
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError, ResourceId,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError, ResourceId,
 };
 
 use data_error::ArklibError;

ark-cli/src/commands/file/utils.rs

@@ -1,6 +1,7 @@
-use crate::error::AppError;
-use crate::models::key_value_to_str;
-use crate::models::Format;
+use crate::{
+    error::AppError,
+    models::{key_value_to_str, Format},
+};
 use data_error::Result as ArklibResult;
 use fs_atomic_versions::atomic::{modify, modify_json, AtomicFile};
 

ark-cli/src/commands/link/utils.rs

@@ -3,8 +3,7 @@ use data_link::Link;
 use std::path::PathBuf;
 use url::Url;
 
-use crate::error::AppError;
-use crate::util::provide_index; // Import your custom AppError type
+use crate::{error::AppError, util::provide_index}; // Import your custom AppError type
 
 pub async fn create_link(
     root: &PathBuf,
@@ -28,8 +27,17 @@ pub fn load_link(
 ) -> Result<Link<ResourceId>, AppError> {
     let path_from_index = id.clone().map(|id| {
         let index = provide_index(root);
-        index.id2path[&id].as_path().to_path_buf()
+        index
+            .get_resources_by_id(&id)
+            .map(|r| r[0].path().to_owned())
+            .ok_or_else(|| {
+                AppError::IndexError(format!(
+                    "Resource with id {} not found",
+                    id
+                ))
+            })
     });
+    let path_from_index = path_from_index.transpose()?;
     let path_from_user = file_path;
 
     let path = match (path_from_user, path_from_index) {
@@ -46,7 +54,7 @@ pub fn load_link(
             }
         }
         (Some(path), None) => Ok(path.to_path_buf()),
-        (None, Some(path)) => Ok(path),
+        (None, Some(path)) => Ok(path.to_path_buf()),
         (None, None) => Err(AppError::LinkLoadError(
             "Provide a path or id for request.".to_owned(),
         ))?,

ark-cli/src/commands/list.rs

@@ -1,5 +1,4 @@
-use std::io::Read;
-use std::path::PathBuf;
+use std::{io::Read, path::PathBuf};
 
 use crate::{
     provide_index, provide_root, read_storage_value, AppError, DateTime,
@@ -76,15 +75,17 @@ impl List {
             .map_err(|_| {
                 AppError::IndexError("Could not read index".to_owned())
             })?
-            .path2id
+            .resources()
             .iter()
-            .filter_map(|(path, resource)| {
+            .filter_map(|indexed_resource| {
+                let path = indexed_resource.path();
+                let id = indexed_resource.id();
                 let tags = if self.tags {
                     Some(
                         read_storage_value(
                             &root,
                             "tags",
-                            &resource.id.to_string(),
+                            &id.to_string(),
                             &None,
                         )
                         .map_or(vec![], |s| {
@@ -102,7 +103,7 @@ impl List {
                         read_storage_value(
                             &root,
                             "scores",
-                            &resource.id.to_string(),
+                            &id.to_string(),
                             &None,
                         )
                         .map_or(0, |s| s.parse::<u32>().unwrap_or(0)),
@@ -114,7 +115,7 @@ impl List {
                 let datetime = if self.modified {
                     let format = "%b %e %H:%M %Y";
                     Some(
-                        DateTime::<Utc>::from(resource.modified)
+                        DateTime::<Utc>::from(indexed_resource.last_modified())
                             .format(format)
                             .to_string(),
                     )
@@ -123,21 +124,18 @@ impl List {
                 };
 
                 let (path, resource, content) = match entry_output {
-                    EntryOutput::Both => (
-                        Some(path.to_owned().into_path_buf()),
-                        Some(resource.clone().id),
-                        None,
-                    ),
-                    EntryOutput::Path => {
-                        (Some(path.to_owned().into_path_buf()), None, None)
+                    EntryOutput::Both => {
+                        (Some(path.to_owned()), Some(id.to_owned()), None)
                     }
-                    EntryOutput::Id => (None, Some(resource.clone().id), None),
+                    EntryOutput::Path => (Some(path.to_owned()), None, None),
+                    EntryOutput::Id => (None, Some(id.to_owned()), None),
                     EntryOutput::Link => match File::open(path) {
                         Ok(mut file) => {
                             let mut contents = String::new();
                             match file.read_to_string(&mut contents) {
                                 Ok(_) => {
-                                    // Check if the content of the file is a valid url
+                                    // Check if the content
+                                    // of the file is a valid url
                                     let url = contents.trim();
                                     let url = url::Url::parse(url);
                                     match url {

ark-cli/src/commands/render.rs

@@ -26,7 +26,8 @@ impl Render {
         let dest_path = filepath.with_file_name(
             filepath
                 .file_stem()
-                // SAFETY: we know that the file stem is valid UTF-8 because it is a file name
+                // SAFETY: we know that the file stem is valid UTF-8
+                // because it is a file name
                 .unwrap()
                 .to_str()
                 .unwrap()

ark-cli/src/commands/storage/list.rs

@@ -1,8 +1,8 @@
 use std::path::PathBuf;
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError,
 };
 
 #[derive(Clone, Debug, clap::Args)]

ark-cli/src/index_registrar.rs

@@ -2,11 +2,13 @@ use lazy_static::lazy_static;
 extern crate canonical_path;
 
 use data_error::{ArklibError, Result};
-use fs_index::ResourceIndex;
+use fs_index::{load_or_build_index, ResourceIndex};
 
-use std::collections::HashMap;
-use std::path::Path;
-use std::sync::{Arc, RwLock};
+use std::{
+    collections::HashMap,
+    path::Path,
+    sync::{Arc, RwLock},
+};
 
 use crate::ResourceId;
 
@@ -34,7 +36,9 @@ pub fn provide_index<P: AsRef<Path>>(
     }
 
     log::info!("Index has not been registered before");
-    match ResourceIndex::provide(&root_path) {
+    // If the index has not been registered before,
+    // we need to load it, update it and register it
+    match load_or_build_index(&root_path, true) {
         Ok(index) => {
             let mut registrar = REGISTRAR.write().map_err(|_| {
                 ArklibError::Other(anyhow::anyhow!("Failed to lock registrar"))

ark-cli/src/main.rs

@@ -1,5 +1,7 @@
-use std::fs::{create_dir_all, File};
-use std::path::PathBuf;
+use std::{
+    fs::{create_dir_all, File},
+    path::PathBuf,
+};
 
 use crate::index_registrar::provide_index;
 use data_pdf::{render_preview_page, PDFQuality};
@@ -15,25 +17,23 @@ use fs_storage::ARK_FOLDER;
 
 use anyhow::Result;
 
-use chrono::prelude::DateTime;
-use chrono::Utc;
+use chrono::{prelude::DateTime, Utc};
 
-use clap::CommandFactory;
-use clap::FromArgMatches;
+use clap::{CommandFactory, FromArgMatches};
 
 use fs_extra::dir::{self, CopyOptions};
 
 use home::home_dir;
 
-use crate::cli::Cli;
-use crate::commands::file::File::{Append, Insert, Read};
-use crate::commands::link::Link::{Create, Load};
-use crate::commands::Commands::Link;
-use crate::commands::Commands::Storage;
-use crate::commands::Commands::*;
-use crate::models::EntryOutput;
-use crate::models::Format;
-use crate::models::Sort;
+use crate::{
+    cli::Cli,
+    commands::{
+        file::File::{Append, Insert, Read},
+        link::Link::{Create, Load},
+        Commands::{Link, Storage, *},
+    },
+    models::{EntryOutput, Format, Sort},
+};
 
 use crate::error::AppError;
 

ark-cli/src/models/storage.rs

@@ -1,7 +1,6 @@
 use crate::ResourceId;
 use fs_atomic_versions::atomic::AtomicFile;
-use std::fmt::Write;
-use std::path::PathBuf;
+use std::{fmt::Write, path::PathBuf};
 
 use crate::{
     commands::{file_append, file_insert, format_file, format_line},

ark-cli/src/util.rs

@@ -1,24 +1,26 @@
 use crate::ResourceId;
-use fs_index::index::ResourceIndex;
+use fs_index::ResourceIndex;
 use fs_metadata::METADATA_STORAGE_FOLDER;
 use fs_properties::PROPERTIES_STORAGE_FOLDER;
 use fs_storage::{
     ARK_FOLDER, PREVIEWS_STORAGE_FOLDER, SCORE_STORAGE_FILE, STATS_FOLDER,
     TAG_STORAGE_FILE, THUMBNAILS_STORAGE_FOLDER,
 };
-use std::env::current_dir;
-use std::fs::{canonicalize, metadata};
-use std::io::BufRead;
-use std::io::BufReader;
-use std::path::Path;
-use std::str::FromStr;
-use std::thread;
-use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
-use std::{fs::File, path::PathBuf};
+use std::{
+    env::current_dir,
+    fs::{canonicalize, metadata, File},
+    io::{BufRead, BufReader},
+    path::{Path, PathBuf},
+    str::FromStr,
+    thread,
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
+};
 
-use crate::error::AppError;
-use crate::models::storage::{Storage, StorageType};
-use crate::ARK_CONFIG;
+use crate::{
+    error::AppError,
+    models::storage::{Storage, StorageType},
+    ARK_CONFIG,
+};
 
 pub fn discover_roots(
     roots_cfg: &Option<PathBuf>,
@@ -111,11 +113,11 @@ pub fn monitor_index(
                             let duration = start.elapsed();
                             println!("Updating succeeded in {:?}\n", duration);
 
-                            if !diff.deleted.is_empty() {
-                                println!("Deleted: {:?}", diff.deleted);
+                            if !diff.removed().is_empty() {
+                                println!("Deleted: {:?}", diff.removed());
                             }
-                            if !diff.added.is_empty() {
-                                println!("Added: {:?}", diff.added);
+                            if !diff.added().is_empty() {
+                                println!("Added: {:?}", diff.added());
                             }
                         }
                     }
@@ -127,10 +129,14 @@ pub fn monitor_index(
                     )
                 })?;
 
-                println!("Here are {} entries in the index", index.size());
+                println!("Here are {} entries in the index", index.len());
 
-                for (key, count) in index.collisions.iter() {
-                    println!("Id {:?} calculated {} times", key, count);
+                for (key, resources) in index.collisions().iter() {
+                    println!(
+                        "Id {:?} calculated {} times",
+                        key,
+                        resources.len()
+                    );
                 }
             }
         }

data-json/src/lib.rs

@@ -1,7 +1,4 @@
-use serde_json::json;
-use serde_json::map::Entry;
-use serde_json::Map;
-use serde_json::Value;
+use serde_json::{json, map::Entry, Map, Value};
 
 pub fn merge(origin: Value, new_data: Value) -> Value {
     match (origin, new_data) {