From e6c9da57ec6cf81d0f45a2db8e1404242a2bd5fb Mon Sep 17 00:00:00 2001 From: 0xZensh Date: Tue, 9 Jul 2024 22:39:32 +0800 Subject: [PATCH] release: v0.4.0, implemented file directory tree. --- src/ic_oss_bucket/README.md | 53 ++++++++++++-- src/ic_oss_bucket/src/api_http.rs | 10 ++- src/ic_oss_bucket/src/api_update.rs | 47 ++++-------- src/ic_oss_bucket/src/store.rs | 110 ++++++++++++++++------------ src/ic_oss_types/src/file.rs | 24 ++++-- 5 files changed, 150 insertions(+), 94 deletions(-) diff --git a/src/ic_oss_bucket/README.md b/src/ic_oss_bucket/README.md index b3fffe4..71f9de6 100644 --- a/src/ic_oss_bucket/README.md +++ b/src/ic_oss_bucket/README.md @@ -21,15 +21,54 @@ dfx deploy ic_oss_bucket --argument "(opt variant {Init = enable_hash_index = true; } })" -# Output: -# ... -# Installing code for canister ic_oss_bucket, with canister ID mmrxu-fqaaa-aaaap-ahhna-cai -# Deployed canisters. -# URLs: -# Backend canister via Candid interface: -# ic_oss_bucket: http://127.0.0.1:4943/?canisterId=bd3sg-teaaa-aaaaa-qaaba-cai&id=mmrxu-fqaaa-aaaap-ahhna-cai + +dfx canister call ic_oss_bucket get_bucket_info '(null)' + +MYID=$(dfx identity get-principal) +ic-oss-cli -i debug/uploader.pem identity +# principal: nprym-ylvyz-ig3fr-lgcmn-zzzt4-tyuix-3v6bm-fsel7-6lq6x-zh2w7-zqe + +dfx canister call ic_oss_bucket admin_set_managers "(vec {principal \"$MYID\"; principal \"nprym-ylvyz-ig3fr-lgcmn-zzzt4-tyuix-3v6bm-fsel7-6lq6x-zh2w7-zqe\"})" + +dfx canister call ic_oss_bucket list_files '(0, null, null, null)' +dfx canister call ic_oss_bucket list_folders '(0, null)' + +ic-oss-cli -i debug/uploader.pem upload -b mmrxu-fqaaa-aaaap-ahhna-cai --file README.md + +dfx canister call ic_oss_bucket get_file_info '(1, null)' + +dfx canister call ic_oss_bucket update_file_info "(record { + id = 1; + status = opt 0; +}, null)" + +dfx canister call ic_oss_bucket create_folder "(record { + parent = 0; + name = \"home\"; +}, null)" +dfx canister call ic_oss_bucket list_folders '(0, null)' + +dfx canister call ic_oss_bucket create_folder "(record { + parent = 1; + name = \"jarvis\"; +}, null)" + +dfx canister call ic_oss_bucket move_file "(record { + id = 1; + from = 0; + to = 2; +}, null)" + +dfx canister call ic_oss_bucket list_files '(2, null, null, null)' + +dfx canister call ic_oss_bucket delete_file '(1, null)' ``` +Download the file in browser: +- by file id: `http://mmrxu-fqaaa-aaaap-ahhna-cai.localhost:4943/f/1` +- by file hash: `http://mmrxu-fqaaa-aaaap-ahhna-cai.localhost:4943/h/b7bb9040d64479a7ca56c8e03ae2daddc819859f7b858488c0b998eeded6dede` + + ## License Copyright © 2024 [LDC Labs](https://github.com/ldclabs). diff --git a/src/ic_oss_bucket/src/api_http.rs b/src/ic_oss_bucket/src/api_http.rs index 1c54117..d772e27 100644 --- a/src/ic_oss_bucket/src/api_http.rs +++ b/src/ic_oss_bucket/src/api_http.rs @@ -176,9 +176,17 @@ fn http_request(request: HttpRequest) -> HttpStreamingResponse { metadata.content_type.clone() }; + let filename = if param.inline { + "" + } else if let Some(ref name) = param.name { + name + } else { + &metadata.name + }; + headers.push(( "content-disposition".to_string(), - content_disposition(&metadata.name), + content_disposition(filename), )); // return all chunks for small file diff --git a/src/ic_oss_bucket/src/api_update.rs b/src/ic_oss_bucket/src/api_update.rs index 70a67b2..6e7f00b 100644 --- a/src/ic_oss_bucket/src/api_update.rs +++ b/src/ic_oss_bucket/src/api_update.rs @@ -56,10 +56,15 @@ fn create_file( store::fs::update_chunk(id, i as u32, now_ms, chunk.to_vec())?; } - if let Some(status) = input.status { - store::fs::update_file(id, |metadata| { - metadata.status = status; - })?; + if input.status.is_some() { + store::fs::update_file( + UpdateFileInput { + id, + status: input.status, + ..Default::default() + }, + now_ms, + )?; } } @@ -98,27 +103,9 @@ fn update_file_info( Ok(()) })?; - let now_ms = ic_cdk::api::time() / MILLISECONDS; - store::fs::update_file(input.id, |metadata| { - if let Some(name) = input.name { - metadata.name = name; - } - if let Some(content_type) = input.content_type { - metadata.content_type = content_type; - } - if let Some(status) = input.status { - metadata.status = status; - } - if input.hash.is_some() { - metadata.hash = input.hash; - } - if input.custom.is_some() { - metadata.custom = input.custom; - } - metadata.updated_at = now_ms; - })?; - - Ok(UpdateFileOutput { updated_at: now_ms }) + let updated_at = ic_cdk::api::time() / MILLISECONDS; + store::fs::update_file(input, updated_at)?; + Ok(UpdateFileOutput { updated_at }) } #[ic_cdk::update(guard = "is_controller_or_manager")] @@ -207,15 +194,7 @@ fn update_folder_info( input.validate()?; let updated_at = ic_cdk::api::time() / MILLISECONDS; - store::fs::update_folder(input.id, |metadata| { - if let Some(name) = input.name { - metadata.name = name; - } - if let Some(status) = input.status { - metadata.status = status; - } - metadata.updated_at = updated_at; - })?; + store::fs::update_folder(input, updated_at)?; Ok(UpdateFolderOutput { updated_at }) } diff --git a/src/ic_oss_bucket/src/store.rs b/src/ic_oss_bucket/src/store.rs index a8095c3..a9a9eb3 100644 --- a/src/ic_oss_bucket/src/store.rs +++ b/src/ic_oss_bucket/src/store.rs @@ -5,8 +5,10 @@ use ic_http_certification::{ HttpCertification, HttpCertificationPath, HttpCertificationTree, HttpCertificationTreeEntry, }; use ic_oss_types::{ - file::{FileChunk, FileInfo, MAX_CHUNK_SIZE, MAX_FILE_SIZE, MAX_FILE_SIZE_PER_CALL}, - folder::{FolderInfo, FolderName}, + file::{ + FileChunk, FileInfo, UpdateFileInput, MAX_CHUNK_SIZE, MAX_FILE_SIZE, MAX_FILE_SIZE_PER_CALL, + }, + folder::{FolderInfo, FolderName, UpdateFolderInput}, ByteN, MapValue, }; use ic_stable_structures::{ @@ -396,7 +398,7 @@ impl FoldersTree { max_children: usize, ) -> Result<(), String> { if id == 0 { - return Err("root folder cannot be moved".to_string()); + Err("root folder cannot be moved".to_string())?; } if from == to { @@ -498,17 +500,17 @@ impl FoldersTree { fn delete_folder(&mut self, id: u32, now_ms: u64) -> Result { if id == 0 { - return Err("root folder cannot be deleted".to_string()); + Err("root folder cannot be deleted".to_string())?; } let parent_id = match self.get(&id) { None => return Ok(false), Some(folder) => { if folder.status > 0 { - return Err("folder is readonly".to_string()); + Err("folder is readonly".to_string())?; } if !folder.folders.is_empty() || !folder.files.is_empty() { - return Err("folder is not empty".to_string()); + Err("folder is not empty".to_string())?; } folder.parent } @@ -716,7 +718,7 @@ pub mod fs { FOLDERS.with(|r| { let id = s.folder_id.saturating_add(1); if id == u32::MAX { - return Err("folder id overflow".to_string()); + Err("folder id overflow".to_string())?; } let mut m = r.borrow_mut(); @@ -739,7 +741,7 @@ pub mod fs { FOLDERS.with(|r| { let id = s.file_id.saturating_add(1); if id == u32::MAX { - return Err("file id overflow".to_string()); + Err("file id overflow".to_string())?; } let mut m = r.borrow_mut(); @@ -750,11 +752,11 @@ pub mod fs { HASHS.with(|r| { let mut m = r.borrow_mut(); if let Some(prev) = m.get(hash.as_ref()) { - return Err(format!("file hash conflict, {}", prev)); + Err(format!("file hash conflict, {}", prev))?; } m.insert(hash.0, id); - Ok(()) + Ok::<(), String>(()) })?; } } @@ -802,17 +804,17 @@ pub mod fs { .ok_or_else(|| format!("file not found: {}", id))?; if file.status != 0 { - return Err(format!("file {} is not writeable", id)); + Err(format!("file {} is not writeable", id))?; } if file.parent != from { - return Err(format!("file {} is not in folder {}", id, from)); + Err(format!("file {} is not in folder {}", id, from))?; } file.parent = to; file.updated_at = now_ms; m.insert(id, file); - Ok(()) + Ok::<(), String>(()) })?; r.borrow_mut().move_file(id, from, to, now_ms); @@ -821,45 +823,61 @@ pub mod fs { }) } - pub fn update_folder( - id: u32, - f: impl FnOnce(&mut FolderMetadata) -> R, - ) -> Result { - if id == 0 { - return Err("root folder cannot be updated".to_string()); + pub fn update_folder(change: UpdateFolderInput, now_ms: u64) -> Result<(), String> { + if change.id == 0 { + Err("root folder cannot be updated".to_string())?; } FOLDERS.with(|r| { let mut m = r.borrow_mut(); - match m.get_mut(&id) { - None => Err(format!("folder not found: {}", id)), + match m.get_mut(&change.id) { + None => Err(format!("folder not found: {}", change.id)), Some(folder) => { - if folder.status > 0 { - return Err("folder is readonly".to_string()); + let status = change.status.unwrap_or(folder.status); + if folder.status > 0 && status > 0 { + Err("folder is readonly".to_string())?; } - - Ok(f(folder)) + if let Some(name) = change.name { + folder.name = name; + } + folder.status = status; + folder.updated_at = now_ms; + Ok(()) } } }) } - pub fn update_file(id: u32, f: impl FnOnce(&mut FileMetadata) -> R) -> Result { + pub fn update_file(change: UpdateFileInput, now_ms: u64) -> Result<(), String> { FS_METADATA.with(|r| { let mut m = r.borrow_mut(); - match m.get(&id) { - None => Err(format!("file not found: {}", id)), + match m.get(&change.id) { + None => Err(format!("file not found: {}", change.id)), Some(mut file) => { let prev_hash = file.hash; - if file.status > 0 { + let status = change.status.unwrap_or(file.status); + if file.status > 0 && status > 0 { Err("file is readonly".to_string())?; } - - let r = f(&mut file); - if file.status == 1 && file.hash.is_none() { + if status == 1 && file.hash.is_none() && change.hash.is_none() { Err("readonly file must have hash".to_string())?; } + file.status = status; + if let Some(name) = change.name { + file.name = name; + } + if let Some(content_type) = change.content_type { + file.content_type = content_type; + } + if change.hash.is_some() { + file.hash = change.hash; + } + if change.custom.is_some() { + file.custom = change.custom; + } + file.updated_at = now_ms; + let enable_hash_index = state::with(|s| s.enable_hash_index); if enable_hash_index && prev_hash != file.hash { HASHS.with(|r| { @@ -868,7 +886,7 @@ pub mod fs { if let Some(prev) = hm.get(&hash.0) { Err(format!("file hash conflict, {}", prev))?; } - hm.insert(hash.0, id); + hm.insert(hash.0, change.id); } if let Some(prev_hash) = prev_hash { hm.remove(&prev_hash.0); @@ -876,8 +894,8 @@ pub mod fs { Ok::<(), String>(()) })?; } - m.insert(id, file); - Ok(r) + m.insert(change.id, file); + Ok(()) } } }) @@ -921,17 +939,17 @@ pub mod fs { None => Err(format!("file not found: {}", id)), Some(file) => { if file.size != file.filled { - return Err("file not fully uploaded".to_string()); + Err("file not fully uploaded".to_string())?; } Ok((file.size, file.chunks)) } })?; if size > MAX_FILE_SIZE.min(usize::MAX as u64) { - return Err(format!( + Err(format!( "file size exceeds limit: {}", MAX_FILE_SIZE.min(usize::MAX as u64) - )); + ))?; } FS_DATA.with(|r| { @@ -950,10 +968,10 @@ pub mod fs { } if filled as u64 != size { - return Err(format!( + Err(format!( "file size mismatch, expected {}, got {}", size, filled - )); + ))?; } Ok(buf) }) @@ -966,14 +984,14 @@ pub mod fs { chunk: Vec, ) -> Result { if chunk.is_empty() { - return Err("empty chunk".to_string()); + Err("empty chunk".to_string())?; } if chunk.len() > MAX_CHUNK_SIZE as usize { - return Err(format!( + Err(format!( "chunk size too large, max size is {} bytes", MAX_CHUNK_SIZE - )); + ))?; } let max = state::with(|s| s.max_file_size); @@ -983,13 +1001,13 @@ pub mod fs { None => Err(format!("file not found: {}", file_id)), Some(mut file) => { if file.status != 0 { - return Err(format!("file {} is not writeable", file_id)); + Err(format!("file {} is not writeable", file_id))?; } file.updated_at = now_ms; file.filled += chunk.len() as u64; if file.filled > max { - panic!("file size exceeds limit: {}", max); + Err(format!("file size exceeds limit: {}", max))?; } match FS_DATA.with(|r| { @@ -1028,7 +1046,7 @@ pub mod fs { match m.get(&id) { Some(file) => { if file.status > 0 { - return Err("file is readonly".to_string()); + Err("file is readonly".to_string())?; } FOLDERS.with(|r| { diff --git a/src/ic_oss_types/src/file.rs b/src/ic_oss_types/src/file.rs index f0d86e1..7bd9ba7 100644 --- a/src/ic_oss_types/src/file.rs +++ b/src/ic_oss_types/src/file.rs @@ -174,17 +174,24 @@ impl UrlFileParam { Url::parse(req_url) }; let url = url.map_err(|_| format!("invalid url: {}", req_url))?; + let mut path_segments = url + .path_segments() + .ok_or_else(|| format!("invalid url path: {}", req_url))?; - let mut param = match url.path() { - path if path.starts_with("/f/") => Self { - file: path[3..].parse().map_err(|_| "invalid file id")?, + let mut param = match path_segments.next() { + Some("f") => Self { + file: path_segments + .next() + .unwrap_or_default() + .parse() + .map_err(|_| "invalid file id")?, hash: None, token: None, name: None, inline: false, }, - path if path.starts_with("/h/") => { - let hash = ByteN::from_hex(&path[3..])?; + Some("h") => { + let hash = ByteN::from_hex(path_segments.next().unwrap_or_default())?; Self { file: 0, hash: Some(hash), @@ -193,7 +200,7 @@ impl UrlFileParam { inline: false, } } - path => return Err(format!("invalid request path: {}", path)), + _ => return Err(format!("invalid url path: {}", req_url)), }; for (key, value) in url.query_pairs() { @@ -215,6 +222,11 @@ impl UrlFileParam { } } + // use the last path segment as filename if provided + if let Some(filename) = path_segments.next() { + param.name = Some(filename.to_string()); + } + Ok(param) } }